feat(admin): add persisted OIDC config APIs (#2267)

Co-authored-by: heihutu <heihutu@gmail.com>

.docker/observability/docker-compose.yml

@@ -15,10 +15,11 @@
 services:
 
   # --- Tracing ---
-
   tempo:
-    image: grafana/tempo:latest
+    image: grafana/tempo:2.10.3
     container_name: tempo
+    depends_on:
+      - redpanda
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
@@ -37,6 +38,38 @@ services:
       timeout: 5s
       retries: 3
       start_period: 15s
+  redpanda:
+    image: redpandadata/redpanda:latest
+    ports:
+      - "9092:9092" # Kafka API for clients
+    command: >
+      redpanda start --overprovisioned
+      --mode=dev-container
+      --kafka-addr=PLAINTEXT://0.0.0.0:9092
+      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
+
+  redpanda-console:
+    image: docker.redpanda.com/redpandadata/console:latest
+    environment:
+      - CONFIG_FILEPATH=/etc/redpanda/redpanda-console-config.yaml
+    volumes:
+      - ./redpanda-console.yaml:/etc/redpanda/redpanda-console-config.yaml
+    ports:
+      - "8080:8080"
+    depends_on:
+      - redpanda
+
+  vulture:
+    image: grafana/tempo-vulture:latest
+    restart: always
+    command:
+      [
+        "-prometheus-listen-address=:8080",
+        "-tempo-query-url=http://tempo:3200",
+        "-tempo-push-url=http://tempo:4317",
+      ]
+    depends_on:
+      - tempo
 
   jaeger:
     image: jaegertracing/jaeger:latest

.docker/observability/grafana/dashboards/rustfs.json

@@ -91,7 +91,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_process_uptime_seconds{job=~\"$job\"}",
+          "expr": "gauge_rustfs_process_uptime_seconds{job=~\"$job\"}",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -223,7 +223,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_buckets_total{job=~\"$job\"})",
+          "expr": "sum(gauge_rustfs_cluster_buckets_total{job=~\"$job\"})",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -289,7 +289,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_objects_total{job=~\"$job\"})",
+          "expr": "sum(gauge_rustfs_cluster_objects_total{job=~\"$job\"})",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -427,7 +427,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"})",
+          "expr": "sum(gauge_rustfs_cluster_capacity_used_bytes{job=~\"$job\"})",
           "legendFormat": "Used",
           "range": true,
           "refId": "A"
@@ -438,7 +438,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
+          "expr": "sum(gauge_rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -450,7 +450,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
+          "expr": "sum(gauge_rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(gauge_rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
           "hide": false,
           "instant": false,
           "legendFormat": "Percent",
@@ -1971,7 +1971,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (drive) (rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"})",
+          "expr": "sum by (drive) (gauge_rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"})",
           "legendFormat": "{{drive}} (bytes)",
           "range": true,
           "refId": "A"
@@ -1982,7 +1982,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (drive) (rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(rustfs_node_disk_total_bytes{job=~\"$job\", drive=~\"$drive\"})",
+          "expr": "sum by (drive) (gauge_rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(gauge_rustfs_node_disk_total_bytes{job=~\"$job\", drive=~\"$drive\"})",
           "hide": false,
           "instant": false,
           "legendFormat": "{{drive}} (percent)",
@@ -2473,7 +2473,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (process_cpu_usage_percent{job=~\"$job\"})",
+          "expr": "sum by (job) (gauge_rustfs_process_cpu_usage{job=~\"$job\"})",
           "legendFormat": "{{job}}",
           "range": true,
           "refId": "A"
@@ -2572,7 +2572,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rustfs_process_memory_bytes{job=~\"$job\"})",
+          "expr": "sum by (job) (gauge_rustfs_process_resident_memory_bytes{job=~\"$job\"})",
           "legendFormat": "{{job}}",
           "range": true,
           "refId": "B"
@@ -2670,7 +2670,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(process_network_io_byte{job=~\"$job\", direction=\"received\"}[5m]))",
+          "expr": "sum by (job) (rate(gauge_rustfs_process_network_io{job=~\"$job\", direction=\"received\"}[5m]))",
           "legendFormat": "RX - {{job}}",
           "range": true,
           "refId": "C"
@@ -2681,7 +2681,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(process_network_io_byte{job=~\"$job\", direction=\"transmitted\"}[5m]))",
+          "expr": "sum by (job) (rate(gauge_rustfs_process_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))",
           "legendFormat": "TX - {{job}}",
           "range": true,
           "refId": "D"
@@ -3339,7 +3339,7 @@
                   "index": 0,
                   "text": "INACTIVE"
                 },
-                "to": 1e-9
+                "to": 1e-09
               },
               "type": "range"
             }
@@ -4048,5 +4048,5 @@
   "timezone": "browser",
   "title": "RustFS",
   "uid": "rustfs-s3",
-  "version": 11
-}
+  "version": 12
+}

.docker/observability/prometheus.yml

@@ -50,6 +50,11 @@ scrape_configs:
     static_configs:
       - targets: [ 'localhost:9090' ]
 
+  - job_name: 'vulture'
+    static_configs:
+      - targets:
+          - 'vulture:8080'
+
 otlp:
   promote_resource_attributes:
     - service.instance.id

.docker/observability/redpanda-console.yaml

@@ -0,0 +1,3 @@
+kafka:
+  brokers:
+    - redpanda:9092

.docker/observability/tempo.yaml

@@ -25,32 +25,18 @@ memberlist:
   join_members:
     - tempo:7946
 
-# Distributor configuration - receives traces and writes directly to ingesters
 distributor:
-  ingester_write_path_enabled: true
-  kafka_write_path_enabled: false
   receivers:
     otlp:
       protocols:
         grpc:
-          endpoint: "tempo:4317"
+          endpoint: "0.0.0.0:4317"
         http:
-          endpoint: "tempo:4318"
-  ring:
-    kvstore:
-      store: memberlist
-
-# Ingester configuration - consumes from Kafka and stores traces
-ingester:
-  lifecycler:
-    ring:
-      kvstore:
-        store: memberlist
-      replication_factor: 1
-    tokens_file_path: /var/tempo/tokens.json
-  trace_idle_period: 10s
-  max_block_bytes: 1_000_000
-  max_block_duration: 5m
+          endpoint: "0.0.0.0:4318"
+  #log_received_spans:
+  #    enabled: true
+  #  log_discarded_spans:
+  #    enabled: true
 
 backend_scheduler:
   provider:
@@ -67,8 +53,7 @@ backend_worker:
       store: memberlist
 
 querier:
-  frontend_worker:
-    frontend_address: tempo:3200
+  query_live_store: true
 
 metrics_generator:
   registry:
@@ -90,9 +75,9 @@ storage:
   trace:
     backend: local
     wal:
-      path: /var/tempo/wal
+      path: /var/tempo/wal # where to store the wal locally
     local:
-      path: /var/tempo/blocks
+      path: /var/tempo/blocks # where to store the traces locally
 
 overrides:
   defaults:
@@ -101,24 +86,14 @@ overrides:
       generate_native_histograms: both
 
 ingest:
-  enabled: false
-  # Disabled because using direct ingester write path
-  # If you want Kafka path, enable this and set:
-  # kafka:
-  #   brokers: [redpanda:9092]
-  #   topic: tempo-ingest
-  #   encoding: protobuf
-  #   consumer_group: tempo-ingest-consumer
+  enabled: true
+  kafka:
+    address: redpanda:9092
+    topic: tempo-ingest
 
 block_builder:
   consume_cycle_duration: 30s
 
-compactor:
-  compaction:
-    block_retention: 168h  # 7 days
-  ring:
-    kvstore:
-      store: memberlist
-
 usage_report:
   reporting_enabled: false
+

.github/dependabot.yml

@@ -26,11 +26,19 @@ updates:
       day: "monday"
       timezone: "Asia/Shanghai"
       time: "08:00"
+    assignees:
+      - "heihutu"
+    reviewers:
+      - "houseme"
+      - "overtrue"
+      - "majinghe"
     ignore:
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"
         versions: [ "0.23.x" ]
+      - dependency-name: "ratelimit"
+        versions: [ "1.x" ]
     groups:
       s3s:
         update-types:

.gitignore

@@ -48,4 +48,5 @@ result*
 *.gz
 rustfs-webdav.code-workspace
 
-.aiexclude
+.aiexclude
+*.bak

Cargo.toml

@@ -148,16 +148,14 @@ rmcp = { version = "1.2.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_bytes = "0.11"
 serde_json = { version = "1.0.149", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
 schemars = "1.2.1"
 
 # Cryptography and Security
 aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
-argon2 = { version = "0.6.0-rc.7" }
+argon2 = { version = "0.6.0-rc.8" }
 blake2 = "0.11.0-rc.5"
-blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
 chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
 hmac = { version = "0.13.0-rc.5" }
@@ -180,7 +178,7 @@ time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros
 
 # Utilities and Tools
 anyhow = "1.0.102"
-arc-swap = "1.8.2"
+arc-swap = "1.9.0"
 astral-tokio-tar = "0.6.0"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
@@ -202,7 +200,7 @@ crossbeam-queue = "0.3.12"
 crossbeam-channel = "0.5.15"
 crossbeam-deque = "0.8.6"
 crossbeam-utils = "0.8.21"
-datafusion = "52.3.0"
+datafusion = "52.4.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
@@ -224,7 +222,7 @@ matchit = "0.9.1"
 md-5 = "0.11.0-rc.5"
 md5 = "0.8.0"
 mime_guess = "2.0.5"
-moka = { version = "0.12.14", features = ["future"] }
+moka = { version = "0.12.15", features = ["future"] }
 netif = "0.1.6"
 num_cpus = { version = "1.17.0" }
 nvml-wrapper = "0.12.0"
@@ -236,7 +234,7 @@ percent-encoding = "2.3.2"
 pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
 rand = { version = "0.10.0", features = ["serde"] }
-ratelimit = "0.10.0"
+ratelimit = "0.10.1"
 rayon = "1.11.0"
 reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
 reed-solomon-simd = "3.1.0"
@@ -269,19 +267,19 @@ transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
 uuid = { version = "1.22.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
-vaultrs = { version = "0.7.4" }
+vaultrs = { version = "0.8.0" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "8.2.0"
+zip = "8.4.0"
 zstd = "0.13.3"
 
 # Observability and Metrics
 metrics = "0.24.3"
 opentelemetry = { version = "0.31.0" }
 opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
-opentelemetry-otlp = { version = "0.31.0", features = ["gzip-http", "reqwest-rustls"] }
+opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] }
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }

Dockerfile

@@ -72,7 +72,8 @@ LABEL name="RustFS" \
       url="https://rustfs.com" \
       license="Apache-2.0"
 
-RUN apk add --no-cache ca-certificates coreutils curl
+RUN apk update && \
+    apk add --no-cache ca-certificates coreutils curl "zlib>=1.3.2-r0"
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs

crates/appauth/Cargo.toml

@@ -33,3 +33,6 @@ rand.workspace = true
 
 [lints]
 workspace = true
+
+[lib]
+doctest = false

crates/audit/Cargo.toml

@@ -47,3 +47,7 @@ rumqttc = { workspace = true }
 
 [lints]
 workspace = true
+
+[lib]
+test = false
+doctest = false

crates/audit/src/registry.rs

@@ -313,6 +313,12 @@ impl AuditRegistry {
                 }
             }
 
+            if &new_config == config {
+                info!("Audit target configuration unchanged, skip persisting server config");
+                info!(count = successful_targets.len(), "All target processing completed");
+                return Ok(successful_targets);
+            }
+
             let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
                 return Err(AuditError::StorageNotAvailable(
                     "Failed to save target configuration: server storage not initialized".to_string(),

crates/audit/tests/performance_test.rs

@@ -159,7 +159,8 @@ async fn test_audit_log_dispatch_performance() {
 
     let start = Instant::now();
 
-    // Dispatch audit log (should be fast since no targets are configured)
+    // Dispatch audit log against an unstarted system state. Empty config keeps
+    // the audit system stopped, so dispatch should fail fast without targets.
     let result = system.dispatch(Arc::new(audit_entry)).await;
     let elapsed = start.elapsed();
 
@@ -168,8 +169,10 @@ async fn test_audit_log_dispatch_performance() {
     // Should be very fast (sub-millisecond for no targets)
     assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {elapsed:?}");
 
-    // Should succeed even with no targets
-    assert!(result.is_ok(), "Dispatch should succeed with no targets");
+    assert!(
+        matches!(result, Err(AuditError::NotInitialized(_))),
+        "Dispatch on a stopped system should return NotInitialized, got: {result:?}"
+    );
 
     // Clean up
     let _ = system.close().await;
@@ -186,11 +189,11 @@ async fn test_system_state_transitions() {
     let config = rustfs_ecstore::config::Config(std::collections::HashMap::new());
     let start_result = system.start(config).await;
 
-    // Should be running (or failed due to server storage)
+    // Empty config keeps the audit system stopped even when start() succeeds.
     let state = system.get_state().await;
     match start_result {
         Ok(_) => {
-            assert_eq!(state, rustfs_audit::system::AuditSystemState::Running);
+            assert_eq!(state, rustfs_audit::system::AuditSystemState::Stopped);
         }
         Err(_) => {
             // Expected in test environment due to server storage not being initialized

crates/audit/tests/system_integration_test.rs

@@ -29,27 +29,21 @@ async fn test_complete_audit_system_lifecycle() {
     assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
     assert!(!system.is_running().await);
 
-    // 2. Start with empty config (will fail due to no server storage in test)
+    // 2. Start with empty config. The current implementation returns Ok(())
+    // but keeps the system stopped when no audit targets are enabled.
     let config = Config(HashMap::new());
     let start_result = system.start(config).await;
 
-    // Should fail in test environment but state handling should work
+    // State handling should remain consistent for both empty-config success and
+    // storage-unavailable failure paths.
     match start_result {
         Err(AuditError::StorageNotAvailable(_)) => {
             // Expected in test environment
             assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
         }
         Ok(_) => {
-            // If it somehow succeeds, verify running state
-            assert_eq!(system.get_state().await, system::AuditSystemState::Running);
-            assert!(system.is_running().await);
-
-            // Test pause/resume
-            system.pause().await.expect("Should pause successfully");
-            assert_eq!(system.get_state().await, system::AuditSystemState::Paused);
-
-            system.resume().await.expect("Should resume successfully");
-            assert_eq!(system.get_state().await, system::AuditSystemState::Running);
+            assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
+            assert!(!system.is_running().await);
         }
         Err(e) => {
             panic!("Unexpected error: {e}");

crates/checksums/Cargo.toml

@@ -36,3 +36,6 @@ sha2 = { workspace = true }
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }
+
+[lib]
+doctest = false

crates/common/Cargo.toml

@@ -41,3 +41,6 @@ rmp-serde = { workspace = true }
 async-trait = { workspace = true }
 s3s = { workspace = true }
 tracing = { workspace = true }
+
+[lib]
+doctest = false

crates/common/src/internode_metrics.rs

@@ -0,0 +1,170 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use metrics::{counter, gauge};
+use std::sync::{
+    Arc, LazyLock,
+    atomic::{AtomicU64, Ordering},
+};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct InternodeMetricsSnapshot {
+    pub sent_bytes_total: u64,
+    pub recv_bytes_total: u64,
+    pub outgoing_requests_total: u64,
+    pub incoming_requests_total: u64,
+    pub errors_total: u64,
+    pub dial_errors_total: u64,
+    pub dial_avg_time_nanos: u64,
+    pub last_dial_unix_millis: u64,
+}
+
+#[derive(Debug, Default)]
+pub struct InternodeMetrics {
+    sent_bytes_total: AtomicU64,
+    recv_bytes_total: AtomicU64,
+    outgoing_requests_total: AtomicU64,
+    incoming_requests_total: AtomicU64,
+    errors_total: AtomicU64,
+    dial_errors_total: AtomicU64,
+    dial_total_time_nanos: AtomicU64,
+    dial_samples_total: AtomicU64,
+    last_dial_unix_millis: AtomicU64,
+}
+
+impl InternodeMetrics {
+    pub fn record_sent_bytes(&self, bytes: usize) {
+        let bytes = bytes as u64;
+        if bytes == 0 {
+            return;
+        }
+        self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        counter!("rustfs.internode.sent.bytes.total").increment(bytes);
+    }
+
+    pub fn record_recv_bytes(&self, bytes: usize) {
+        let bytes = bytes as u64;
+        if bytes == 0 {
+            return;
+        }
+        self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        counter!("rustfs.internode.recv.bytes.total").increment(bytes);
+    }
+
+    pub fn record_outgoing_request(&self) {
+        self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.requests.outgoing.total").increment(1);
+    }
+
+    pub fn record_incoming_request(&self) {
+        self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.requests.incoming.total").increment(1);
+    }
+
+    pub fn record_error(&self) {
+        self.errors_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.errors.total").increment(1);
+    }
+
+    pub fn record_dial_result(&self, duration: Duration, success: bool) {
+        let elapsed_nanos = duration.as_nanos().min(u128::from(u64::MAX)) as u64;
+        self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed);
+        let samples = self.dial_samples_total.fetch_add(1, Ordering::Relaxed) + 1;
+        let total = self.dial_total_time_nanos.load(Ordering::Relaxed);
+        gauge!("rustfs.internode.dial.avg_time.nanos").set(total as f64 / samples as f64);
+
+        if !success {
+            self.dial_errors_total.fetch_add(1, Ordering::Relaxed);
+            counter!("rustfs.internode.dial.errors.total").increment(1);
+        }
+
+        let now_ms = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_millis()
+            .min(u128::from(u64::MAX)) as u64;
+        self.last_dial_unix_millis.store(now_ms, Ordering::Relaxed);
+    }
+
+    pub fn snapshot(&self) -> InternodeMetricsSnapshot {
+        let dial_samples_total = self.dial_samples_total.load(Ordering::Relaxed);
+        let dial_total_time_nanos = self.dial_total_time_nanos.load(Ordering::Relaxed);
+        let dial_avg_time_nanos = if dial_samples_total == 0 {
+            0
+        } else {
+            dial_total_time_nanos / dial_samples_total
+        };
+
+        InternodeMetricsSnapshot {
+            sent_bytes_total: self.sent_bytes_total.load(Ordering::Relaxed),
+            recv_bytes_total: self.recv_bytes_total.load(Ordering::Relaxed),
+            outgoing_requests_total: self.outgoing_requests_total.load(Ordering::Relaxed),
+            incoming_requests_total: self.incoming_requests_total.load(Ordering::Relaxed),
+            errors_total: self.errors_total.load(Ordering::Relaxed),
+            dial_errors_total: self.dial_errors_total.load(Ordering::Relaxed),
+            dial_avg_time_nanos,
+            last_dial_unix_millis: self.last_dial_unix_millis.load(Ordering::Relaxed),
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn reset_for_test(&self) {
+        self.sent_bytes_total.store(0, Ordering::Relaxed);
+        self.recv_bytes_total.store(0, Ordering::Relaxed);
+        self.outgoing_requests_total.store(0, Ordering::Relaxed);
+        self.incoming_requests_total.store(0, Ordering::Relaxed);
+        self.errors_total.store(0, Ordering::Relaxed);
+        self.dial_errors_total.store(0, Ordering::Relaxed);
+        self.dial_total_time_nanos.store(0, Ordering::Relaxed);
+        self.dial_samples_total.store(0, Ordering::Relaxed);
+        self.last_dial_unix_millis.store(0, Ordering::Relaxed);
+    }
+}
+
+pub fn global_internode_metrics() -> &'static Arc<InternodeMetrics> {
+    static GLOBAL_INTERNODE_METRICS: LazyLock<Arc<InternodeMetrics>> = LazyLock::new(|| Arc::new(InternodeMetrics::default()));
+    &GLOBAL_INTERNODE_METRICS
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn snapshot_reports_recorded_values() {
+        let metrics = global_internode_metrics();
+        metrics.reset_for_test();
+
+        metrics.record_sent_bytes(64);
+        metrics.record_recv_bytes(32);
+        metrics.record_outgoing_request();
+        metrics.record_incoming_request();
+        metrics.record_error();
+        metrics.record_dial_result(Duration::from_millis(9), true);
+        metrics.record_dial_result(Duration::from_millis(3), false);
+
+        let snapshot = metrics.snapshot();
+        assert_eq!(snapshot.sent_bytes_total, 64);
+        assert_eq!(snapshot.recv_bytes_total, 32);
+        assert_eq!(snapshot.outgoing_requests_total, 1);
+        assert_eq!(snapshot.incoming_requests_total, 1);
+        assert_eq!(snapshot.errors_total, 1);
+        assert_eq!(snapshot.dial_errors_total, 1);
+        assert_eq!(snapshot.dial_avg_time_nanos, 6_000_000);
+        assert!(snapshot.last_dial_unix_millis > 0);
+
+        metrics.reset_for_test();
+    }
+}

crates/common/src/lib.rs

@@ -17,6 +17,7 @@ pub mod bucket_stats;
 pub mod data_usage;
 pub mod globals;
 pub mod heal_channel;
+pub mod internode_metrics;
 pub mod last_minute;
 pub mod metrics;
 mod readiness;

crates/common/src/readiness.rs

@@ -60,6 +60,23 @@ impl GlobalReadiness {
     pub fn is_ready(&self) -> bool {
         self.status.load(Ordering::SeqCst) == SystemStage::FullReady as u8
     }
+
+    /// Get the current system stage
+    /// # Returns
+    /// The current SystemStage of the service
+    pub fn current_stage(&self) -> SystemStage {
+        match self.status.load(Ordering::SeqCst) {
+            0 => SystemStage::Booting,
+            1 => SystemStage::StorageReady,
+            2 => SystemStage::IamReady,
+            3 => SystemStage::FullReady,
+            invalid => {
+                debug_assert!(false, "GlobalReadiness::current_stage: invalid status value {}", invalid);
+                // Fallback to the most conservative stage on invalid values
+                SystemStage::Booting
+            }
+        }
+    }
 }
 
 #[cfg(test)]

crates/config/Cargo.toml

@@ -37,3 +37,6 @@ constants = ["dep:const-str"]
 notify = ["dep:const-str", "constants"]
 observability = ["constants"]
 opa = ["constants"]
+
+[lib]
+doctest = false

crates/config/src/constants/app.rs

@@ -131,9 +131,61 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 /// Environment variable for server volumes.
 pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
 
+/// Environment variable for server access key.
+pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
+
+/// Environment variable for server access key file.
+pub const ENV_RUSTFS_ACCESS_KEY_FILE: &str = "RUSTFS_ACCESS_KEY_FILE";
+
+/// Environment variable for server root user.
+pub const ENV_RUSTFS_ROOT_USER: &str = "RUSTFS_ROOT_USER";
+
+/// Environment variable for server secret key.
+pub const ENV_RUSTFS_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
+
+/// Environment variable for server secret key file.
+pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE";
+
+/// Environment variable for server root password.
+pub const ENV_RUSTFS_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD";
+
+/// Environment variable for server OBS endpoint.
+pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
+
+/// Environment variable for console server enable.
+pub const ENV_RUSTFS_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE";
+
+/// Environment variable for console server address.
+pub const ENV_RUSTFS_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS";
+
 /// Environment variable for server tls path.
 pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH";
 
+/// Environment variable for server KMS enable.
+pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
+
+/// Default KMS enable for server-side encryption
+/// This is the default value for enabling KMS encryption for server-side encryption.
+/// Default value: false
+pub const DEFAULT_KMS_ENABLE: bool = false;
+
+/// Environment variable for server KMS backend.
+pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND";
+
+/// Default KMS backend for server-side encryption
+/// This is the default KMS backend for server-side encryption.
+/// Default value: local
+pub const DEFAULT_KMS_BACKEND: &str = "local";
+
+/// Environment variable for selecting the buffer profile used for adaptive buffer sizing.
+pub const ENV_RUSTFS_BUFFER_PROFILE: &str = "RUSTFS_BUFFER_PROFILE";
+
+/// Default buffer profile for adaptive buffer sizing
+/// This is the default buffer profile for adaptive buffer sizing.
+/// It is used to identify the workload profile for adaptive buffer sizing.
+/// Default value: GeneralPurpose
+pub const DEFAULT_BUFFER_PROFILE: &str = "GeneralPurpose";
+
 /// Default value for the server TLS path if `ENV_RUSTFS_TLS_PATH` is not set.
 pub const DEFAULT_RUSTFS_TLS_PATH: &str = "";
 

crates/config/src/constants/mod.rs

@@ -28,3 +28,4 @@ pub(crate) mod runtime;
 pub(crate) mod scanner;
 pub(crate) mod targets;
 pub(crate) mod tls;
+pub(crate) mod workload;

crates/config/src/constants/object.rs

@@ -180,3 +180,265 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
 
 /// Default: bitrot verification is enabled on GetObject reads (do not skip).
 pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
+
+// =============================================================================
+// Concurrent Request Fix - Timeout and Backpressure Configuration
+// =============================================================================
+
+/// Environment variable for GetObject request timeout in seconds.
+///
+/// When a GetObject request exceeds this duration, it will be cancelled
+/// and return a 504 Gateway Timeout error. This prevents requests from
+/// hanging indefinitely due to deadlocks or resource exhaustion.
+///
+/// Default: 30 seconds (can be overridden by `RUSTFS_OBJECT_GET_TIMEOUT`).
+/// Set to 0 to disable timeout (not recommended for production).
+pub const ENV_OBJECT_GET_TIMEOUT: &str = "RUSTFS_OBJECT_GET_TIMEOUT";
+
+/// Default GetObject request timeout in seconds.
+///
+/// This value balances between allowing large object transfers to complete
+/// and preventing indefinite hangs. For 20-26MB objects with concurrent
+/// range reads, 30 seconds should be sufficient under normal conditions.
+pub const DEFAULT_OBJECT_GET_TIMEOUT: u64 = 30;
+
+/// Environment variable for disk read operation timeout in seconds.
+///
+/// Individual disk read operations that exceed this duration will be
+/// cancelled and treated as failures. This helps detect slow or hung
+/// disks without waiting indefinitely.
+///
+/// Default: 10 seconds (can be overridden by `RUSTFS_OBJECT_DISK_READ_TIMEOUT`).
+pub const ENV_OBJECT_DISK_READ_TIMEOUT: &str = "RUSTFS_OBJECT_DISK_READ_TIMEOUT";
+
+/// Default disk read timeout in seconds.
+pub const DEFAULT_OBJECT_DISK_READ_TIMEOUT: u64 = 10;
+
+/// Environment variable for duplex pipe buffer size in bytes.
+///
+/// The duplex pipe connects the disk read task to the HTTP response stream.
+/// A larger buffer reduces backpressure but increases memory usage.
+/// For large objects (20-26MB), a 4MB buffer provides good throughput.
+///
+/// Default: 4194304 (4 MB, can be overridden by `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE`).
+/// Minimum recommended: 1048576 (1 MB).
+pub const ENV_OBJECT_DUPLEX_BUFFER_SIZE: &str = "RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE";
+
+/// Default duplex buffer size: 4 MB.
+///
+/// This is 4x larger than the original 1 MB buffer, providing better
+/// handling of large objects and reducing backpressure-related hangs.
+pub const DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE: usize = 4 * 1024 * 1024;
+
+/// Environment variable for I/O buffer size in bytes.
+///
+/// This controls the buffer size used for individual I/O operations.
+/// A larger buffer improves throughput for sequential reads but may
+/// increase latency for small random reads.
+///
+/// Default: 131072 (128 KB, can be overridden by `RUSTFS_OBJECT_IO_BUFFER_SIZE`).
+pub const ENV_OBJECT_IO_BUFFER_SIZE: &str = "RUSTFS_OBJECT_IO_BUFFER_SIZE";
+
+/// Default I/O buffer size: 128 KB.
+pub const DEFAULT_OBJECT_IO_BUFFER_SIZE: usize = 128 * 1024;
+
+/// Environment variable to enable/disable lock optimization.
+///
+/// When enabled, read locks are released immediately after metadata
+/// is read, rather than being held for the entire data transfer.
+/// This significantly reduces lock contention under high concurrency.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE`).
+pub const ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE: &str = "RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE";
+
+/// Default: lock optimization is enabled.
+pub const DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE: bool = true;
+
+/// Environment variable to enable/disable priority-based I/O scheduling.
+///
+/// When enabled, smaller requests (< 1MB) are given higher priority
+/// than larger requests (> 10MB), preventing "starvation" of small
+/// requests by large ones.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE`).
+pub const ENV_OBJECT_PRIORITY_SCHEDULING_ENABLE: &str = "RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE";
+
+/// Default: priority scheduling is enabled.
+pub const DEFAULT_OBJECT_PRIORITY_SCHEDULING_ENABLE: bool = true;
+
+/// Environment variable to enable/disable deadlock detection.
+///
+/// When enabled, the system monitors active requests and detects
+/// potential deadlock situations (circular lock wait chains).
+/// This has some performance overhead and is intended for debugging.
+///
+/// Default: false (disabled, can be overridden by `RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE`).
+pub const ENV_OBJECT_DEADLOCK_DETECTION_ENABLE: &str = "RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE";
+
+/// Default: deadlock detection is disabled for performance.
+pub const DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE: bool = false;
+
+/// Environment variable for deadlock detection check interval in seconds.
+///
+/// How often the deadlock detector analyzes the lock wait graph.
+/// More frequent checks detect deadlocks faster but use more CPU.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL`).
+pub const ENV_OBJECT_DEADLOCK_CHECK_INTERVAL: &str = "RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL";
+
+/// Default deadlock check interval: 5 seconds.
+pub const DEFAULT_OBJECT_DEADLOCK_CHECK_INTERVAL: u64 = 5;
+
+/// Environment variable for deadlock detection hang threshold in seconds.
+///
+/// Requests that have been running longer than this threshold are
+/// considered "potentially hung" and included in deadlock analysis.
+///
+/// Default: 10 seconds (can be overridden by `RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD`).
+pub const ENV_OBJECT_DEADLOCK_HANG_THRESHOLD: &str = "RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD";
+
+/// Default hang threshold: 10 seconds.
+pub const DEFAULT_OBJECT_DEADLOCK_HANG_THRESHOLD: u64 = 10;
+
+/// Environment variable for backpressure high watermark percentage.
+///
+/// When buffer usage exceeds this percentage, the system enters
+/// "high watermark" state and may apply backpressure to producers.
+///
+/// Default: 80 (80%, can be overridden by `RUSTFS_OBJECT_BACKPRESSURE_HIGH_WATERMARK`).
+pub const ENV_OBJECT_BACKPRESSURE_HIGH_WATERMARK: &str = "RUSTFS_OBJECT_BACKPRESSURE_HIGH_WATERMARK";
+
+/// Default high watermark: 80%.
+pub const DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK: u32 = 80;
+
+/// Environment variable for backpressure low watermark percentage.
+///
+/// When buffer usage drops below this percentage after being in
+/// high watermark state, backpressure is released.
+///
+/// Default: 50 (50%, can be overridden by `RUSTFS_OBJECT_BACKPRESSURE_LOW_WATERMARK`).
+pub const ENV_OBJECT_BACKPRESSURE_LOW_WATERMARK: &str = "RUSTFS_OBJECT_BACKPRESSURE_LOW_WATERMARK";
+
+/// Default low watermark: 50%.
+pub const DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK: u32 = 50;
+
+/// Environment variable for lock acquisition timeout in seconds.
+///
+/// When a lock cannot be acquired within this duration, the operation
+/// will fail with a timeout error. This prevents indefinite waiting
+/// for locks that may never be released due to deadlocks.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT`).
+pub const ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT: &str = "RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT";
+
+/// Default lock acquisition timeout: 5 seconds.
+pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5;
+
+// ============================================================================
+// I/O priority scheduling configuration
+// ============================================================================
+
+/// Environment variable for I/O high priority size threshold in bytes.
+///
+/// Requests smaller than this threshold are classified as high priority.
+/// High priority requests are processed first to prevent starvation of small requests.
+///
+/// Default: 1048576 (1 MB, can be overridden by `RUSTFS_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD`).
+pub const ENV_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD: &str = "RUSTFS_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD";
+
+/// Default high priority size threshold: 1 MB.
+pub const DEFAULT_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD: usize = 1024 * 1024;
+
+/// Environment variable for I/O low priority size threshold in bytes.
+///
+/// Requests larger than this threshold are classified as low priority.
+/// Low priority requests are processed last to avoid blocking small requests.
+///
+/// Default: 104857600 (100 MB, can be overridden by `RUSTFS_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD`).
+pub const ENV_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD: &str = "RUSTFS_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD";
+
+/// Default low priority size threshold: 100 MB.
+pub const DEFAULT_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD: usize = 100 * 1024 * 1024;
+
+/// Environment variable for high priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the high priority queue.
+///
+/// Default: 32 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_HIGH_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_HIGH_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_HIGH_CAPACITY";
+
+/// Default high priority queue capacity: 32.
+pub const DEFAULT_OBJECT_IO_QUEUE_HIGH_CAPACITY: usize = 32;
+
+/// Environment variable for normal priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the normal priority queue.
+///
+/// Default: 64 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_NORMAL_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_NORMAL_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_NORMAL_CAPACITY";
+
+/// Default normal priority queue capacity: 64.
+pub const DEFAULT_OBJECT_IO_QUEUE_NORMAL_CAPACITY: usize = 64;
+
+/// Environment variable for low priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the low priority queue.
+///
+/// Default: 16 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_LOW_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_LOW_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_LOW_CAPACITY";
+
+/// Default low priority queue capacity: 16.
+pub const DEFAULT_OBJECT_IO_QUEUE_LOW_CAPACITY: usize = 16;
+
+/// Environment variable for starvation prevention check interval in milliseconds.
+///
+/// How often the system checks for starving low-priority requests.
+/// When a low-priority request has been waiting longer than the starvation threshold,
+/// it is promoted to normal priority.
+///
+/// Default: 100 ms (can be overridden by `RUSTFS_OBJECT_IO_STARVATION_PREVENTION_INTERVAL`).
+pub const ENV_OBJECT_IO_STARVATION_PREVENTION_INTERVAL: &str = "RUSTFS_OBJECT_IO_STARVATION_PREVENTION_INTERVAL";
+
+/// Default starvation prevention interval: 100 ms.
+pub const DEFAULT_OBJECT_IO_STARVATION_PREVENTION_INTERVAL: u64 = 100;
+
+/// Environment variable for starvation threshold in seconds.
+///
+/// Maximum time a low-priority request can wait before being promoted to normal priority.
+/// This prevents indefinite starvation of low-priority requests.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_IO_STARVATION_THRESHOLD_SECS`).
+pub const ENV_OBJECT_IO_STARVATION_THRESHOLD_SECS: &str = "RUSTFS_OBJECT_IO_STARVATION_THRESHOLD_SECS";
+
+/// Default starvation threshold: 5 seconds.
+pub const DEFAULT_OBJECT_IO_STARVATION_THRESHOLD_SECS: u64 = 5;
+
+/// Environment variable for load sampling window size.
+///
+/// Number of recent samples used to calculate I/O load metrics.
+///
+/// Default: 100 samples (can be overridden by `RUSTFS_OBJECT_IO_LOAD_SAMPLE_WINDOW`).
+pub const ENV_OBJECT_IO_LOAD_SAMPLE_WINDOW: &str = "RUSTFS_OBJECT_IO_LOAD_SAMPLE_WINDOW";
+
+/// Default load sampling window: 100 samples.
+pub const DEFAULT_OBJECT_IO_LOAD_SAMPLE_WINDOW: usize = 100;
+
+/// Environment variable for high load wait time threshold in milliseconds.
+///
+/// When average wait time exceeds this threshold, the system is considered to be under high load.
+///
+/// Default: 50 ms (can be overridden by `RUSTFS_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS`).
+pub const ENV_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS: &str = "RUSTFS_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS";
+
+/// Default high load threshold: 50 ms.
+pub const DEFAULT_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS: u64 = 50;
+
+/// Environment variable for low load wait time threshold in milliseconds.
+///
+/// When average wait time is below this threshold, the system is considered to be under low load.
+///
+/// Default: 10 ms (can be overridden by `RUSTFS_OBJECT_IO_LOAD_LOW_THRESHOLD_MS`).
+pub const ENV_OBJECT_IO_LOAD_LOW_THRESHOLD_MS: &str = "RUSTFS_OBJECT_IO_LOAD_LOW_THRESHOLD_MS";
+
+/// Default low load threshold: 10 ms.
+pub const DEFAULT_OBJECT_IO_LOAD_LOW_THRESHOLD_MS: u64 = 10;

crates/config/src/constants/workload.rs

@@ -0,0 +1,70 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Workload profile buffer configuration constants.
+//!
+//! This module defines environment variable keys and default values for
+//! custom buffer profile configuration.
+
+use crate::{KI_B, MI_B};
+
+/// Environment variable for minimum buffer size
+/// Default: 64KB (65536 bytes)
+pub const ENV_RUSTFS_BUFFER_MIN_SIZE: &str = "RUSTFS_BUFFER_MIN_SIZE";
+
+/// Environment variable for maximum buffer size
+/// Default: 1MB (1048576 bytes)
+pub const ENV_RUSTFS_BUFFER_MAX_SIZE: &str = "RUSTFS_BUFFER_MAX_SIZE";
+
+/// Environment variable for default buffer size (used when file size is unknown)
+/// Default: 256KB (262144 bytes)
+pub const ENV_RUSTFS_BUFFER_DEFAULT_SIZE: &str = "RUSTFS_BUFFER_DEFAULT_SIZE";
+
+/// Default minimum buffer size: 64KB
+pub const DEFAULT_BUFFER_MIN_SIZE: usize = 64 * KI_B;
+
+/// Default maximum buffer size: 1MB
+pub const DEFAULT_BUFFER_MAX_SIZE: usize = MI_B;
+
+/// Default buffer size for unknown file size: 256KB
+pub const DEFAULT_BUFFER_UNKNOWN_SIZE: usize = 256 * KI_B;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::MI_B;
+
+    #[test]
+    fn test_default_values() {
+        assert_eq!(DEFAULT_BUFFER_MIN_SIZE, 65536); // 64KB
+        assert_eq!(DEFAULT_BUFFER_MAX_SIZE, 1048576); // 1MB
+        assert_eq!(DEFAULT_BUFFER_UNKNOWN_SIZE, 262144); // 256KB
+    }
+
+    #[test]
+    fn test_constants() {
+        assert_eq!(KI_B, 1024);
+        assert_eq!(MI_B, 1024 * 1024);
+        assert_eq!(64 * KI_B, DEFAULT_BUFFER_MIN_SIZE);
+        assert_eq!(MI_B, DEFAULT_BUFFER_MAX_SIZE);
+        assert_eq!(256 * KI_B, DEFAULT_BUFFER_UNKNOWN_SIZE);
+    }
+
+    #[test]
+    fn test_env_var_names() {
+        assert_eq!(ENV_RUSTFS_BUFFER_MIN_SIZE, "RUSTFS_BUFFER_MIN_SIZE");
+        assert_eq!(ENV_RUSTFS_BUFFER_MAX_SIZE, "RUSTFS_BUFFER_MAX_SIZE");
+        assert_eq!(ENV_RUSTFS_BUFFER_DEFAULT_SIZE, "RUSTFS_BUFFER_DEFAULT_SIZE");
+    }
+}

crates/config/src/lib.rs

@@ -45,6 +45,8 @@ pub use constants::targets::*;
 #[cfg(feature = "constants")]
 pub use constants::tls::*;
 #[cfg(feature = "constants")]
+pub use constants::workload::*;
+#[cfg(feature = "constants")]
 pub mod oidc {
     pub use super::constants::oidc::*;
 }

crates/crypto/Cargo.toml

@@ -56,3 +56,6 @@ crypto = [
     "dep:rand",
     "dep:sha2",
 ]
+
+[lib]
+doctest = false

crates/e2e_test/src/delete_objects_versioning_test.rs

@@ -16,15 +16,13 @@
 //! "In a versioned Bucket, DeleteMarkers are not appearing straight after
 //! a delete_objects is called."
 //!
-//! Root cause: `delete_versions_internal` wrote new xl.meta to disk via
-//! `write_all_private` without invalidating the `GlobalFileCache`. Subsequent
-//! calls to `read_metadata` returned the stale cached xl.meta (without the
-//! delete marker), making `list_object_versions` show the old version as
-//! `IsLatest=true` rather than the new delete marker.
+//! Root cause: metadata updates could become temporarily invisible to
+//! `list_object_versions`, so the old version was still reported as
+//! `IsLatest=true` instead of the newly-created delete marker.
 //!
-//! Fix: `write_all_private` now calls `get_global_file_cache().invalidate()`
-//! after every successful write, and `rename_data` also invalidates the cache
-//! for the destination path after the atomic rename.
+//! Fix: metadata write, delete, and rename paths now make the updated
+//! `xl.meta` immediately visible, and the old file-cache shortcut has been
+//! removed from the read path.
 
 #[cfg(test)]
 mod tests {

crates/e2e_test/src/lib.rs

@@ -66,6 +66,11 @@ mod compression_test;
 // Regression test for Issue #1878: DeleteMarkers not visible immediately after delete_objects
 #[cfg(test)]
 mod delete_objects_versioning_test;
+
+// Regression test for Issue #2252: ListObjectVersions misses newest version after put -> delete -> put
+#[cfg(test)]
+mod list_object_versions_regression_test;
+
 #[cfg(test)]
 mod protocols;
 

crates/e2e_test/src/list_object_versions_regression_test.rs

@@ -0,0 +1,182 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for Issue #2252:
+//! "ListObjectVersions misses the newest version after create -> delete -> create."
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use serial_test::serial;
+    use tracing::info;
+
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_immediately_returns_latest_put_after_delete_marker() {
+        init_logging();
+        info!("🧪 TEST: ListObjectVersions returns the newest version immediately after put -> delete -> put");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-object-versions-2252";
+        let key = "test-prefix/test-object.txt";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to enable versioning");
+
+        let first_put = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"first version"))
+            .send()
+            .await
+            .expect("Failed to put first object version");
+        let first_version_id = first_put
+            .version_id()
+            .map(str::to_string)
+            .expect("First put should return a version_id");
+
+        let delete_resp = client
+            .delete_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("Failed to create delete marker");
+        let delete_marker_version_id = delete_resp
+            .version_id()
+            .map(str::to_string)
+            .expect("DeleteObject should return a delete marker version_id");
+
+        let second_put = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"second version"))
+            .send()
+            .await
+            .expect("Failed to put second object version");
+        let second_version_id = second_put
+            .version_id()
+            .map(str::to_string)
+            .expect("Second put should return a version_id");
+
+        let first_listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions immediately after second put");
+        let second_listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions a second time");
+
+        let first_versions = first_listing.versions().to_vec();
+        let first_delete_markers = first_listing.delete_markers().to_vec();
+        let second_versions = second_listing.versions().to_vec();
+        let second_delete_markers = second_listing.delete_markers().to_vec();
+
+        info!(
+            "First listing: {} versions, {} delete markers",
+            first_versions.len(),
+            first_delete_markers.len()
+        );
+        info!(
+            "Second listing: {} versions, {} delete markers",
+            second_versions.len(),
+            second_delete_markers.len()
+        );
+
+        assert_eq!(
+            first_versions.len(),
+            2,
+            "First ListObjectVersions call should return both object versions immediately (regression #2252)"
+        );
+        assert_eq!(
+            first_delete_markers.len(),
+            1,
+            "First ListObjectVersions call should return the delete marker immediately (regression #2252)"
+        );
+        assert_eq!(
+            second_versions.len(),
+            2,
+            "Second ListObjectVersions call should still return both object versions"
+        );
+        assert_eq!(
+            second_delete_markers.len(),
+            1,
+            "Second ListObjectVersions call should still return the delete marker"
+        );
+
+        let first_latest_version = first_versions
+            .iter()
+            .find(|version| version.version_id() == Some(second_version_id.as_str()))
+            .expect("First listing should include the newest object version");
+        assert_eq!(
+            first_latest_version.is_latest(),
+            Some(true),
+            "Newest object version should be latest on the first listing"
+        );
+
+        let first_original_version = first_versions
+            .iter()
+            .find(|version| version.version_id() == Some(first_version_id.as_str()))
+            .expect("First listing should include the original object version");
+        assert_eq!(
+            first_original_version.is_latest(),
+            Some(false),
+            "Original object version should no longer be latest"
+        );
+
+        let first_delete_marker = first_delete_markers
+            .iter()
+            .find(|marker| marker.version_id() == Some(delete_marker_version_id.as_str()))
+            .expect("First listing should include the delete marker");
+        assert_eq!(
+            first_delete_marker.is_latest(),
+            Some(false),
+            "Delete marker should no longer be latest after the second put"
+        );
+    }
+}

crates/e2e_test/src/reliant/grpc_lock_client.rs

@@ -61,6 +61,7 @@ impl GrpcLockClient {
             metadata: LockMetadata::default(),
             priority: LockPriority::Normal,
             deadlock_detection: false,
+            suppress_contention_logs: false,
         }
     }
 }

crates/ecstore/Cargo.toml

@@ -74,7 +74,6 @@ regex = { workspace = true }
 path-absolutize = { workspace = true }
 rmp.workspace = true
 rmp-serde.workspace = true
-serde_bytes.workspace = true
 tokio-util = { workspace = true, features = ["io", "compat"] }
 base64 = { workspace = true }
 hmac = { workspace = true }
@@ -117,6 +116,9 @@ faster-hex = { workspace = true }
 ratelimit = { workspace = true }
 aws-smithy-http-client.workspace = true
 
+# Observability and Metrics
+metrics = { workspace = true }
+
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 criterion = { workspace = true, features = ["html_reports"] }
@@ -134,3 +136,6 @@ harness = false
 [[bench]]
 name = "comparison_benchmark"
 harness = false
+
+[lib]
+doctest = false

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -33,7 +33,9 @@ use crate::global::GLOBAL_LocalNodeName;
 use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
 use crate::store::ECStore;
 use crate::store_api::StorageAPI;
-use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete};
+use crate::store_api::{
+    GetObjectReader, HTTPRangeSpec, ListOperations, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete,
+};
 use crate::tier::warm_backend::WarmBackendGetOpts;
 use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
 use bytes::BytesMut;
@@ -646,18 +648,48 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
 }
 
 pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
-    let lc = GLOBAL_LifecycleSys.get(&oi.bucket).await;
-    if !lc.is_none() {
-        let event = lc.expect("err").eval(&oi.to_lifecycle_opts()).await;
-        match event.action {
-            IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
-                if oi.delete_marker || oi.is_dir {
-                    return;
-                }
-                GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await;
-            }
-            _ => (),
+    if let Some(lc) = GLOBAL_LifecycleSys.get(&oi.bucket).await {
+        enqueue_transition_with_lifecycle(oi, &lc, &src).await;
+    }
+}
+
+pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
+    let Some(lc) = GLOBAL_LifecycleSys.get(bucket).await else {
+        return Ok(());
+    };
+    let mut marker = None;
+    let mut version_marker = None;
+    let src = LcEventSrc::Scanner;
+
+    loop {
+        let page = api
+            .clone()
+            .list_object_versions(bucket, "", marker.clone(), version_marker.clone(), None, 1000)
+            .await?;
+
+        for object in &page.objects {
+            enqueue_transition_with_lifecycle(object, &lc, &src).await;
         }
+
+        if !page.is_truncated {
+            return Ok(());
+        }
+
+        marker = page.next_marker;
+        version_marker = page.next_version_idmarker;
+    }
+}
+
+async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &BucketLifecycleConfiguration, src: &LcEventSrc) {
+    let event = lc.eval(&oi.to_lifecycle_opts()).await;
+    match event.action {
+        IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
+            if oi.delete_marker || oi.is_dir {
+                return;
+            }
+            GLOBAL_TransitionState.queue_transition_task(oi, &event, src).await;
+        }
+        _ => (),
     }
 }
 

crates/ecstore/src/bucket/lifecycle/lifecycle.rs

@@ -768,7 +768,7 @@ impl LifecycleCalculate for NoncurrentVersionTransition {
 #[async_trait::async_trait]
 impl LifecycleCalculate for Transition {
     fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
-        if !obj.is_latest || self.days.is_none() {
+        if !obj.is_latest {
             return None;
         }
 
@@ -1296,6 +1296,45 @@ mod tests {
         assert_eq!(event.storage_class, "COLDTIER44");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn eval_inner_transitions_latest_object_after_date_due() {
+        let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
+        let transition_date = base_time - Duration::days(1);
+        let lc = BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: None,
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: None,
+                filter: None,
+                id: Some("transition-date".to_string()),
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: Some(vec![Transition {
+                    days: None,
+                    date: Some(transition_date.into()),
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]),
+            }],
+        };
+
+        let opts = ObjectOpts {
+            name: "obj".to_string(),
+            mod_time: Some(base_time),
+            is_latest: true,
+            transition_status: "".to_string(),
+            ..Default::default()
+        };
+        let event = lc.eval_inner(&opts, base_time + Duration::days(1), 0).await;
+
+        assert_eq!(event.action, IlmAction::TransitionAction);
+        assert_eq!(event.rule_id, "transition-date");
+        assert_eq!(event.storage_class, "WARM");
+    }
+
     #[tokio::test]
     #[serial]
     async fn eval_inner_expires_noncurrent_version_after_due() {

crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs

@@ -51,6 +51,10 @@ impl LastDayTierStats {
         self.bins[now_idx] = self.bins[now_idx].add(&ts);
     }
 
+    pub fn total(&self) -> TierStats {
+        self.bins.iter().fold(TierStats::default(), |acc, bin| acc.add(bin))
+    }
+
     fn forward_to(&mut self, t: &mut OffsetDateTime) {
         if t.unix_timestamp() == 0 {
             *t = OffsetDateTime::now_utc();
@@ -99,4 +103,30 @@ impl LastDayTierStats {
 }
 
 #[cfg(test)]
-mod test {}
+mod test {
+    use super::*;
+
+    #[test]
+    fn total_sums_all_recorded_stats() {
+        let mut stats = LastDayTierStats::default();
+        stats.add_stats(TierStats {
+            total_size: 10,
+            num_versions: 1,
+            num_objects: 1,
+        });
+        stats.add_stats(TierStats {
+            total_size: 20,
+            num_versions: 2,
+            num_objects: 0,
+        });
+
+        assert_eq!(
+            stats.total(),
+            TierStats {
+                total_size: 30,
+                num_versions: 3,
+                num_objects: 1,
+            }
+        );
+    }
+}

crates/ecstore/src/bucket/metadata.rs

@@ -33,7 +33,7 @@ use serde::Serializer;
 use std::collections::HashMap;
 use std::io::{Read, Write};
 use std::sync::Arc;
-use time::OffsetDateTime;
+use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset};
 use tracing::error;
 
 fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
@@ -43,19 +43,177 @@ fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
     Ok(String::from_utf8(buf)?)
 }
 
+fn read_msgp_bool<R: Read>(rd: &mut R) -> Result<bool> {
+    let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    match marker {
+        rmp::Marker::True => Ok(true),
+        rmp::Marker::False => Ok(false),
+        rmp::Marker::FixPos(v) => Ok(v != 0),
+        rmp::Marker::U8 => Ok(read_u8(rd)? != 0),
+        rmp::Marker::U16 => Ok(read_u16_raw(rd)? != 0),
+        rmp::Marker::U32 => Ok(read_u32_raw(rd)? != 0),
+        rmp::Marker::U64 => Ok(read_u64_raw(rd)? != 0),
+        rmp::Marker::I8 => Ok(read_i8_raw(rd)? != 0),
+        rmp::Marker::I16 => Ok(read_i16_raw(rd)? != 0),
+        rmp::Marker::I32 => Ok(read_i32_raw(rd)? != 0),
+        rmp::Marker::I64 => Ok(read_i64_raw(rd)? != 0),
+        rmp::Marker::FixNeg(v) => Ok(v != 0),
+        _ => Err(Error::other(format!("expected bool or int-like bool, got marker: {marker:?}"))),
+    }
+}
+
 fn read_msgp_time_value<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
     let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
     match marker {
         rmp::Marker::Null => Ok(OffsetDateTime::UNIX_EPOCH),
         rmp::Marker::Ext8 => read_msgp_ext8_time(rd),
+        rmp::Marker::FixArray(len) => read_msgp_legacy_compact_time(rd, u32::from(len)),
+        rmp::Marker::Array16 => {
+            let len = read_u16_raw(rd)?;
+            read_msgp_legacy_compact_time(rd, u32::from(len))
+        }
+        rmp::Marker::Array32 => {
+            let len = read_u32_raw(rd)?;
+            read_msgp_legacy_compact_time(rd, len)
+        }
+        rmp::Marker::Bin8 => {
+            let len = usize::from(read_u8(rd)?);
+            read_msgp_time_value_from_embedded_bin(rd, len)
+        }
+        rmp::Marker::Bin16 => {
+            let len = usize::from(read_u16_raw(rd)?);
+            read_msgp_time_value_from_embedded_bin(rd, len)
+        }
+        rmp::Marker::Bin32 => {
+            let len = read_u32_raw(rd)? as usize;
+            read_msgp_time_value_from_embedded_bin(rd, len)
+        }
         _ => Err(Error::other(format!("expected time ext or nil, got marker: {marker:?}"))),
     }
 }
 
-fn read_msgp_bin<R: Read>(rd: &mut R) -> Result<Vec<u8>> {
-    let len = rmp::decode::read_bin_len(rd)? as usize;
+fn read_u8<R: Read>(rd: &mut R) -> Result<u8> {
+    let mut buf = [0u8; 1];
+    rd.read_exact(&mut buf)?;
+    Ok(buf[0])
+}
+
+fn read_u16_raw<R: Read>(rd: &mut R) -> Result<u16> {
+    let mut buf = [0u8; 2];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_u16(&buf))
+}
+
+fn read_u32_raw<R: Read>(rd: &mut R) -> Result<u32> {
+    let mut buf = [0u8; 4];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_u32(&buf))
+}
+
+fn read_u64_raw<R: Read>(rd: &mut R) -> Result<u64> {
+    let mut buf = [0u8; 8];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_u64(&buf))
+}
+
+fn read_i8_raw<R: Read>(rd: &mut R) -> Result<i8> {
+    Ok(read_u8(rd)? as i8)
+}
+
+fn read_i16_raw<R: Read>(rd: &mut R) -> Result<i16> {
+    let mut buf = [0u8; 2];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_i16(&buf))
+}
+
+fn read_i32_raw<R: Read>(rd: &mut R) -> Result<i32> {
+    let mut buf = [0u8; 4];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_i32(&buf))
+}
+
+fn read_i64_raw<R: Read>(rd: &mut R) -> Result<i64> {
+    let mut buf = [0u8; 8];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_i64(&buf))
+}
+
+fn read_msgp_time_value_from_embedded_bin<R: Read>(rd: &mut R, len: usize) -> Result<OffsetDateTime> {
     let mut buf = vec![0u8; len];
     rd.read_exact(&mut buf)?;
+    let mut cur = std::io::Cursor::new(buf);
+    read_msgp_time_value(&mut cur)
+}
+
+fn read_msgp_legacy_compact_time<R: Read>(rd: &mut R, len: u32) -> Result<OffsetDateTime> {
+    if len != 9 {
+        return Err(Error::other(format!("invalid legacy compact time len: {len}")));
+    }
+
+    let year: i32 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let ordinal: u16 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let hour: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let minute: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let second: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let nanosecond: u32 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let offset_hour: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let offset_minute: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let offset_second: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+
+    let date =
+        Date::from_ordinal_date(year, ordinal).map_err(|e| Error::other(format!("invalid legacy compact time date: {e}")))?;
+    let time = CivilTime::from_hms_nano(hour, minute, second, nanosecond)
+        .map_err(|e| Error::other(format!("invalid legacy compact time time: {e}")))?;
+    let offset = UtcOffset::from_hms(offset_hour, offset_minute, offset_second)
+        .map_err(|e| Error::other(format!("invalid legacy compact time offset: {e}")))?;
+
+    Ok(PrimitiveDateTime::new(date, time)
+        .assume_offset(offset)
+        .to_offset(UtcOffset::UTC))
+}
+
+fn read_msgp_bin<R: Read>(rd: &mut R) -> Result<Vec<u8>> {
+    let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    match marker {
+        rmp::Marker::Null => Ok(Vec::new()),
+        rmp::Marker::Bin8 => {
+            let len = usize::from(read_u8(rd)?);
+            read_exact_bytes(rd, len)
+        }
+        rmp::Marker::Bin16 => {
+            let len = usize::from(read_u16_raw(rd)?);
+            read_exact_bytes(rd, len)
+        }
+        rmp::Marker::Bin32 => {
+            let len = read_u32_raw(rd)? as usize;
+            read_exact_bytes(rd, len)
+        }
+        rmp::Marker::FixArray(len) => read_msgp_legacy_byte_array(rd, u32::from(len)),
+        rmp::Marker::Array16 => {
+            let len = read_u16_raw(rd)?;
+            read_msgp_legacy_byte_array(rd, u32::from(len))
+        }
+        rmp::Marker::Array32 => {
+            let len = read_u32_raw(rd)?;
+            read_msgp_legacy_byte_array(rd, len)
+        }
+        _ => Err(Error::other(format!("expected bin or byte array, got marker: {marker:?}"))),
+    }
+}
+
+fn read_exact_bytes<R: Read>(rd: &mut R, len: usize) -> Result<Vec<u8>> {
+    let mut buf = vec![0u8; len];
+    rd.read_exact(&mut buf)?;
+    Ok(buf)
+}
+
+fn read_msgp_legacy_byte_array<R: Read>(rd: &mut R, len: u32) -> Result<Vec<u8>> {
+    let mut buf = Vec::with_capacity(len as usize);
+    for _ in 0..len {
+        let value: i64 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+        let byte = u8::try_from(value).map_err(|_| Error::other(format!("byte value out of range: {value}")))?;
+        buf.push(byte);
+    }
     Ok(buf)
 }
 
@@ -227,18 +385,20 @@ impl BucketMetadata {
             match key.as_str() {
                 "Name" => self.name = read_msgp_str(rd)?,
                 "Created" => self.created = read_msgp_time_value(rd)?,
-                "LockEnabled" => self.lock_enabled = rmp::decode::read_bool(rd)?,
-                "PolicyConfigJSON" => self.policy_config_json = read_msgp_bin(rd)?,
-                "NotificationConfigXML" => self.notification_config_xml = read_msgp_bin(rd)?,
-                "LifecycleConfigXML" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
-                "ObjectLockConfigXML" => self.object_lock_config_xml = read_msgp_bin(rd)?,
-                "VersioningConfigXML" => self.versioning_config_xml = read_msgp_bin(rd)?,
-                "EncryptionConfigXML" => self.encryption_config_xml = read_msgp_bin(rd)?,
-                "TaggingConfigXML" => self.tagging_config_xml = read_msgp_bin(rd)?,
-                "QuotaConfigJSON" => self.quota_config_json = read_msgp_bin(rd)?,
-                "ReplicationConfigXML" => self.replication_config_xml = read_msgp_bin(rd)?,
-                "BucketTargetsConfigJSON" => self.bucket_targets_config_json = read_msgp_bin(rd)?,
-                "BucketTargetsConfigMetaJSON" => self.bucket_targets_config_meta_json = read_msgp_bin(rd)?,
+                "LockEnabled" => self.lock_enabled = read_msgp_bool(rd)?,
+                "PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?,
+                "NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?,
+                "LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
+                "ObjectLockConfigXML" | "ObjectLockConfigXml" => self.object_lock_config_xml = read_msgp_bin(rd)?,
+                "VersioningConfigXML" | "VersioningConfigXml" => self.versioning_config_xml = read_msgp_bin(rd)?,
+                "EncryptionConfigXML" | "EncryptionConfigXml" => self.encryption_config_xml = read_msgp_bin(rd)?,
+                "TaggingConfigXML" | "TaggingConfigXml" => self.tagging_config_xml = read_msgp_bin(rd)?,
+                "QuotaConfigJSON" | "QuotaConfigJson" => self.quota_config_json = read_msgp_bin(rd)?,
+                "ReplicationConfigXML" | "ReplicationConfigXml" => self.replication_config_xml = read_msgp_bin(rd)?,
+                "BucketTargetsConfigJSON" | "BucketTargetsConfigJson" => self.bucket_targets_config_json = read_msgp_bin(rd)?,
+                "BucketTargetsConfigMetaJSON" | "BucketTargetsConfigMetaJson" => {
+                    self.bucket_targets_config_meta_json = read_msgp_bin(rd)?
+                }
                 "PolicyConfigUpdatedAt" => self.policy_config_updated_at = read_msgp_time_value(rd)?,
                 "ObjectLockConfigUpdatedAt" => self.object_lock_config_updated_at = read_msgp_time_value(rd)?,
                 "EncryptionConfigUpdatedAt" => self.encryption_config_updated_at = read_msgp_time_value(rd)?,
@@ -250,9 +410,11 @@ impl BucketMetadata {
                 "NotificationConfigUpdatedAt" => self.notification_config_updated_at = read_msgp_time_value(rd)?,
                 "BucketTargetsConfigUpdatedAt" => self.bucket_targets_config_updated_at = read_msgp_time_value(rd)?,
                 "BucketTargetsConfigMetaUpdatedAt" => self.bucket_targets_config_meta_updated_at = read_msgp_time_value(rd)?,
-                "CorsConfigXML" => self.cors_config_xml = read_msgp_bin(rd)?,
-                "PublicAccessBlockConfigXML" => self.public_access_block_config_xml = read_msgp_bin(rd)?,
-                "BucketAclConfigJSON" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
+                "CorsConfigXML" | "CorsConfigXml" => self.cors_config_xml = read_msgp_bin(rd)?,
+                "PublicAccessBlockConfigXML" | "PublicAccessBlockConfigXml" => {
+                    self.public_access_block_config_xml = read_msgp_bin(rd)?
+                }
+                "BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
                 "CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
                 "PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?,
                 "BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,

crates/ecstore/src/bucket/metadata_test.rs

@@ -98,6 +98,143 @@ async fn unmarshal_test_bucket_metadata() {
     assert!(bm.bucket_acl_config_json.is_empty());
 }
 
+#[test]
+fn unmarshal_legacy_compact_time_bucket_metadata() {
+    use faster_hex::hex_decode;
+
+    let legacy_hex = concat!(
+        "83",
+        "a44e616d65",
+        "a474657374",
+        "a743726561746564",
+        "99cd07e9cd01100c1021ce2026b1fa000000",
+        "ab4c6f636b456e61626c6564",
+        "c2"
+    );
+
+    let mut bytes = vec![0u8; legacy_hex.len() / 2];
+    hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
+
+    let bm = BucketMetadata::unmarshal(&bytes).expect("legacy compact time should decode");
+
+    assert_eq!(bm.name, "test");
+    assert_eq!(bm.created.unix_timestamp(), 1759148193);
+    assert_eq!(bm.created.nanosecond(), 539406842);
+    assert!(!bm.lock_enabled);
+}
+
+#[test]
+fn unmarshal_bin_wrapped_ext_time_bucket_metadata() {
+    use faster_hex::hex_decode;
+
+    let wrapped_hex = concat!(
+        "83",
+        "a44e616d65",
+        "a464616461",
+        "a743726561746564",
+        "c40fc70c05fffffff1886e090000000000",
+        "ab4c6f636b456e61626c6564",
+        "c2"
+    );
+
+    let mut bytes = vec![0u8; wrapped_hex.len() / 2];
+    hex_decode(wrapped_hex.as_bytes(), &mut bytes).expect("valid hex");
+
+    let bm = BucketMetadata::unmarshal(&bytes).expect("bin-wrapped ext time should decode");
+
+    assert_eq!(bm.name, "dada");
+    assert_eq!(bm.created.unix_timestamp(), -62135596800);
+    assert!(!bm.lock_enabled);
+}
+
+#[test]
+fn unmarshal_legacy_rmp_serde_field_aliases_and_byte_arrays() {
+    use faster_hex::hex_decode;
+
+    let legacy_hex = concat!(
+        "85",
+        "a44e616d65",
+        "a474657374",
+        "a743726561746564",
+        "99cd07e9cd01100c1021ce2026b1fa000000",
+        "ab4c6f636b456e61626c6564",
+        "c2",
+        "b0506f6c696379436f6e6669674a736f6e",
+        "93010203",
+        "bb4275636b657454617267657473436f6e6669674d6574614a736f6e",
+        "920405"
+    );
+
+    let mut bytes = vec![0u8; legacy_hex.len() / 2];
+    hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
+
+    let bm = BucketMetadata::unmarshal(&bytes).expect("legacy field aliases and byte arrays should decode");
+
+    assert_eq!(bm.name, "test");
+    assert_eq!(bm.created.unix_timestamp(), 1759148193);
+    assert_eq!(bm.created.nanosecond(), 539406842);
+    assert_eq!(bm.policy_config_json, vec![1, 2, 3]);
+    assert_eq!(bm.bucket_targets_config_meta_json, vec![4, 5]);
+    assert!(!bm.lock_enabled);
+}
+
+#[test]
+fn unmarshal_legacy_bin16_and_array16_bucket_metadata() {
+    let policy = vec![b'x'; 257];
+    let targets_meta: Vec<u8> = (0u8..=16).collect();
+
+    let mut bytes = Vec::new();
+    rmp::encode::write_map_len(&mut bytes, 5).unwrap();
+    rmp::encode::write_str(&mut bytes, "Name").unwrap();
+    rmp::encode::write_str(&mut bytes, "test-bucket").unwrap();
+    rmp::encode::write_str(&mut bytes, "Created").unwrap();
+    bytes.extend_from_slice(&[
+        0xc7, 0x0c, 0x05, 0x00, 0x00, 0x00, 0x00, 0x65, 0x92, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
+    ]);
+    rmp::encode::write_str(&mut bytes, "LockEnabled").unwrap();
+    bytes.push(0x01);
+    rmp::encode::write_str(&mut bytes, "PolicyConfigJson").unwrap();
+    rmp::encode::write_bin(&mut bytes, &policy).unwrap();
+    rmp::encode::write_str(&mut bytes, "BucketTargetsConfigMetaJson").unwrap();
+    rmp::encode::write_array_len(&mut bytes, targets_meta.len() as u32).unwrap();
+    for byte in &targets_meta {
+        rmp::encode::write_uint(&mut bytes, u64::from(*byte)).unwrap();
+    }
+
+    let bm = BucketMetadata::unmarshal(&bytes).expect("legacy bin16 and array16 should decode");
+
+    assert_eq!(bm.name, "test-bucket");
+    assert_eq!(bm.created.unix_timestamp(), 1704067200);
+    assert!(bm.lock_enabled);
+    assert_eq!(bm.policy_config_json, policy);
+    assert_eq!(bm.bucket_targets_config_meta_json, targets_meta);
+}
+
+#[test]
+fn unmarshal_legacy_numeric_bool_bucket_metadata() {
+    use faster_hex::hex_decode;
+
+    let legacy_hex = concat!(
+        "83",
+        "a44e616d65",
+        "ab746573742d6275636b6574",
+        "a743726561746564",
+        "c70c05000000006592008000000000",
+        "ab4c6f636b456e61626c6564",
+        "01"
+    );
+
+    let mut bytes = vec![0u8; legacy_hex.len() / 2];
+    hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
+
+    let bm = BucketMetadata::unmarshal(&bytes).expect("legacy numeric bool should decode");
+
+    assert_eq!(bm.name, "test-bucket");
+    assert_eq!(bm.created.unix_timestamp(), 1704067200);
+    assert_eq!(bm.created.nanosecond(), 0);
+    assert!(bm.lock_enabled);
+}
+
 #[tokio::test]
 async fn marshal_msg_complete_example() {
     // Create a complete BucketMetadata with various configurations

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2453,7 +2453,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             return rinfo;
         }
 
-        let sopts = StatObjectOptions {
+        let mut sopts = StatObjectOptions {
             version_id: object_info.version_id.map(|v| v.to_string()).unwrap_or_default(),
             internal: AdvancedGetOptions {
                 replication_proxy_request: "false".to_string(),
@@ -2462,7 +2462,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             ..Default::default()
         };
 
-        sopts.set(AMZ_TAGGING_DIRECTIVE, "ACCESS");
+        if let Err(err) = sopts.set(AMZ_TAGGING_DIRECTIVE, "ACCESS") {
+            warn!("failed to set replication tagging directive header: {err}");
+        }
 
         match tgt_client
             .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))

crates/ecstore/src/client/api_get_options.rs

@@ -64,20 +64,32 @@ impl GetObjectOptions {
     pub fn header(&self) -> HeaderMap {
         let mut headers: HeaderMap = HeaderMap::with_capacity(self.headers.len());
         for (k, v) in &self.headers {
-            if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
-                headers.insert(header_name, v.parse().expect("err"));
-            } else {
-                warn!("Invalid header name: {}", k);
+            match (HeaderName::from_bytes(k.as_bytes()), HeaderValue::from_str(v)) {
+                (Ok(header_name), Ok(header_value)) => {
+                    headers.insert(header_name, header_value);
+                }
+                (Err(_), _) => {
+                    warn!("Invalid header name: {}", k);
+                }
+                (_, Err(_)) => {
+                    warn!("Invalid header value for {}: {:?}", k, v);
+                }
             }
         }
         if self.checksum {
-            headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err"));
+            headers.insert(HeaderName::from_static("x-amz-checksum-mode"), HeaderValue::from_static("ENABLED"));
         }
         headers
     }
 
-    pub fn set(&self, key: &str, value: &str) {
-        //self.headers[http.CanonicalHeaderKey(key)] = value;
+    pub fn set(&mut self, key: &str, value: &str) -> Result<(), std::io::Error> {
+        let header_name = HeaderName::from_bytes(key.as_bytes())
+            .map_err(|err| std::io::Error::other(err_invalid_argument(&format!("Invalid header name {key}: {err}"))))?;
+        HeaderValue::from_str(value)
+            .map_err(|err| std::io::Error::other(err_invalid_argument(&format!("Invalid header value for {key}: {err}"))))?;
+
+        self.headers.insert(header_name.as_str().to_string(), value.to_string());
+        Ok(())
     }
 
     pub fn set_req_param(&mut self, key: &str, value: &str) {
@@ -89,12 +101,12 @@ impl GetObjectOptions {
     }
 
     pub fn set_match_etag(&mut self, etag: &str) -> Result<(), std::io::Error> {
-        self.set("If-Match", &format!("\"{etag}\""));
+        self.set("If-Match", &format!("\"{etag}\""))?;
         Ok(())
     }
 
     pub fn set_match_etag_except(&mut self, etag: &str) -> Result<(), std::io::Error> {
-        self.set("If-None-Match", &format!("\"{etag}\""));
+        self.set("If-None-Match", &format!("\"{etag}\""))?;
         Ok(())
     }
 
@@ -102,7 +114,7 @@ impl GetObjectOptions {
         if mod_time.unix_timestamp() == 0 {
             return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
         }
-        self.set("If-Unmodified-Since", &mod_time.to_string());
+        self.set("If-Unmodified-Since", &mod_time.to_string())?;
         Ok(())
     }
 
@@ -110,17 +122,17 @@ impl GetObjectOptions {
         if mod_time.unix_timestamp() == 0 {
             return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
         }
-        self.set("If-Modified-Since", &mod_time.to_string());
+        self.set("If-Modified-Since", &mod_time.to_string())?;
         Ok(())
     }
 
     pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> {
         if start == 0 && end < 0 {
-            self.set("Range", &format!("bytes={}", end));
+            self.set("Range", &format!("bytes={}", end))?;
         } else if 0 < start && end == 0 {
-            self.set("Range", &format!("bytes={}-", start));
+            self.set("Range", &format!("bytes={}-", start))?;
         } else if 0 <= start && start <= end {
-            self.set("Range", &format!("bytes={}-{}", start, end));
+            self.set("Range", &format!("bytes={}-{}", start, end))?;
         } else {
             return Err(std::io::Error::other(err_invalid_argument(&format!(
                 "Invalid range specified: start={} end={}",
@@ -146,3 +158,40 @@ impl GetObjectOptions {
         url_values
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::GetObjectOptions;
+
+    #[test]
+    fn set_range_populates_range_header() {
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(5, 9).expect("valid range should succeed");
+
+        let headers = opts.header();
+        let range = headers.get("range").expect("range header should be present");
+        assert_eq!(range.to_str().expect("range header must be valid ascii"), "bytes=5-9");
+    }
+
+    #[test]
+    fn set_rejects_invalid_header_value() {
+        let mut opts = GetObjectOptions::default();
+
+        let err = opts
+            .set("Range", "bytes=5-\n9")
+            .expect_err("invalid header value should fail");
+
+        assert!(err.to_string().contains("Invalid header value"));
+        assert!(opts.headers.is_empty(), "invalid headers must not be stored");
+    }
+
+    #[test]
+    fn header_skips_invalid_prepopulated_header_value() {
+        let mut opts = GetObjectOptions::default();
+        opts.headers.insert("Range".to_string(), "bytes=5-\n9".to_string());
+
+        let headers = opts.header();
+
+        assert!(headers.get("range").is_none(), "invalid stored header values should be ignored");
+    }
+}

crates/ecstore/src/client/transition_api.rs

@@ -375,7 +375,7 @@ impl TransitionClient {
         //debug!("http_resp_body: {}", String::from_utf8(b).unwrap());
 
         //if self.is_trace_enabled && !(self.trace_errors_only && resp.status() == StatusCode::OK) {
-        if resp.status() != StatusCode::OK {
+        if !resp.status().is_success() {
             //self.dump_http(&cloned_req, &resp)?;
             let mut body_vec = Vec::new();
             let mut body = resp.into_body();

crates/ecstore/src/config/com.rs

@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::config::{Config, GLOBAL_STORAGE_CLASS, storageclass};
+use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, oidc, storageclass};
 use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result};
+use crate::global::is_first_cluster_node_local;
 use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
 use http::HeaderMap;
-use rustfs_config::{DEFAULT_DELIMITER, RUSTFS_REGION};
+use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
+use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
 use rustfs_utils::path::SLASH_SEPARATOR;
 use serde_json::{Map, Value};
 use std::collections::{HashMap, HashSet};
@@ -44,6 +46,19 @@ pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u
     Ok(data)
 }
 
+pub async fn read_config_no_lock<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
+    let (data, _obj) = read_config_with_metadata(
+        api,
+        file,
+        &ObjectOptions {
+            no_lock: true,
+            ..Default::default()
+        },
+    )
+    .await?;
+    Ok(data)
+}
+
 pub async fn read_config_with_metadata<S: StorageAPI>(
     api: Arc<S>,
     file: &str,
@@ -167,6 +182,85 @@ fn parse_inline_block_value(value: &Value) -> Option<String> {
     }
 }
 
+fn parse_oidc_scalar_value(key: &str, value: &Value) -> Option<String> {
+    match value {
+        Value::String(v) => Some(v.trim().to_string()),
+        Value::Bool(v) if key == ENABLE_KEY || key == OIDC_REDIRECT_URI_DYNAMIC => Some(if *v {
+            EnableState::On.to_string()
+        } else {
+            EnableState::Off.to_string()
+        }),
+        Value::Bool(v) => Some(v.to_string()),
+        Value::Number(v) => Some(v.to_string()),
+        Value::Array(values) if key == rustfs_config::oidc::OIDC_SCOPES => {
+            let scopes = values
+                .iter()
+                .filter_map(Value::as_str)
+                .map(str::trim)
+                .filter(|scope| !scope.is_empty())
+                .collect::<Vec<_>>()
+                .join(",");
+            Some(scopes)
+        }
+        Value::Null => None,
+        _ => None,
+    }
+}
+
+fn decode_oidc_provider_object(provider: &Map<String, Value>) -> KVS {
+    let mut kvs = oidc::DEFAULT_IDENTITY_OPENID_KVS.clone();
+
+    for (key, value) in provider {
+        if !IDENTITY_OPENID_KEYS.contains(&key.as_str()) || key == COMMENT_KEY {
+            continue;
+        }
+
+        if let Some(parsed) = parse_oidc_scalar_value(key, value) {
+            kvs.insert(key.clone(), parsed);
+        }
+    }
+
+    kvs
+}
+
+fn apply_external_oidc_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
+    let oidc_root = root.get("openid").or_else(|| root.get(IDENTITY_OPENID_SUB_SYS));
+    let Some(Value::Object(oidc_obj)) = oidc_root else {
+        return false;
+    };
+
+    if oidc_obj.is_empty() {
+        return false;
+    }
+
+    let subsystem = cfg.0.entry(IDENTITY_OPENID_SUB_SYS.to_string()).or_default();
+    let mut applied = false;
+
+    for (raw_instance, provider) in oidc_obj {
+        let instance_key = if raw_instance == "default" {
+            DEFAULT_DELIMITER.to_string()
+        } else {
+            raw_instance.to_string()
+        };
+
+        match provider {
+            Value::Object(provider_obj) => {
+                subsystem.insert(instance_key, decode_oidc_provider_object(provider_obj));
+                applied = true;
+            }
+            Value::Array(_) => {
+                if let Ok(kvs) = serde_json::from_value::<KVS>(provider.clone()) {
+                    subsystem.insert(instance_key, kvs);
+                    applied = true;
+                }
+            }
+            _ => {}
+        }
+    }
+
+    applied
+}
+
 fn apply_external_storage_class_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
     let sc = root.get("storageclass").or_else(|| root.get("storage_class"));
     let Some(Value::Object(sc_obj)) = sc else {
@@ -210,8 +304,9 @@ fn decode_server_config_blob(data: &[u8]) -> Result<Config> {
 
     let mut cfg = Config::new();
     let has_storage = apply_external_storage_class_map(&mut cfg, &root);
+    let has_oidc = apply_external_oidc_map(&mut cfg, &root);
     let has_header = root.contains_key("version") || root.contains_key("region") || root.contains_key("credential");
-    if !has_storage && !has_header {
+    if !has_storage && !has_oidc && !has_header {
         return Err(Error::other("unrecognized external server config shape"));
     }
     Ok(cfg)
@@ -241,6 +336,119 @@ fn build_storageclass_object(cfg: &Config) -> Map<String, Value> {
     sc_obj
 }
 
+fn build_oidc_provider_object(kvs: &KVS) -> Map<String, Value> {
+    let mut provider = Map::new();
+
+    for kv in &kvs.0 {
+        if kv.key == COMMENT_KEY || (kv.hidden_if_empty && kv.value.trim().is_empty()) {
+            continue;
+        }
+
+        if kv.value.trim().is_empty() {
+            continue;
+        }
+
+        if kv.key == ENABLE_KEY || kv.key == OIDC_REDIRECT_URI_DYNAMIC {
+            let enabled = kv
+                .value
+                .parse::<EnableState>()
+                .map(|state| state.is_enabled())
+                .unwrap_or(false);
+            provider.insert(kv.key.clone(), Value::Bool(enabled));
+            continue;
+        }
+
+        if kv.key == rustfs_config::oidc::OIDC_SCOPES {
+            let scopes = kv
+                .value
+                .split(',')
+                .map(str::trim)
+                .filter(|scope| !scope.is_empty())
+                .map(|scope| Value::String(scope.to_string()))
+                .collect::<Vec<_>>();
+            provider.insert(kv.key.clone(), Value::Array(scopes));
+            continue;
+        }
+
+        provider.insert(kv.key.clone(), Value::String(kv.value.clone()));
+    }
+
+    provider
+}
+
+fn build_oidc_object(cfg: &Config) -> Map<String, Value> {
+    let Some(subsystem) = cfg.0.get(IDENTITY_OPENID_SUB_SYS) else {
+        return Map::new();
+    };
+
+    let mut providers = subsystem.iter().collect::<Vec<_>>();
+    providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+
+    let mut oidc_obj = Map::new();
+    for (instance_key, kvs) in providers {
+        if kvs
+            .lookup(rustfs_config::oidc::OIDC_CONFIG_URL)
+            .unwrap_or_default()
+            .trim()
+            .is_empty()
+        {
+            continue;
+        }
+
+        let provider = build_oidc_provider_object(kvs);
+        if provider.is_empty() {
+            continue;
+        }
+
+        let external_key = if instance_key == DEFAULT_DELIMITER {
+            "default".to_string()
+        } else {
+            instance_key.clone()
+        };
+        oidc_obj.insert(external_key, Value::Object(provider));
+    }
+
+    oidc_obj
+}
+
+fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
+    let Some(subsystem) = cfg.0.get(IDENTITY_OPENID_SUB_SYS) else {
+        return Map::new();
+    };
+
+    let mut providers = subsystem.iter().collect::<Vec<_>>();
+    providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+
+    let mut oidc_obj = Map::new();
+    for (instance_key, kvs) in providers {
+        let mut normalized = oidc::DEFAULT_IDENTITY_OPENID_KVS.clone();
+        normalized.extend(kvs.clone());
+
+        if normalized
+            .lookup(rustfs_config::oidc::OIDC_CONFIG_URL)
+            .unwrap_or_default()
+            .trim()
+            .is_empty()
+        {
+            continue;
+        }
+
+        let provider = build_oidc_provider_object(&normalized);
+        if provider.is_empty() {
+            continue;
+        }
+
+        let external_key = if instance_key == DEFAULT_DELIMITER {
+            "default".to_string()
+        } else {
+            instance_key.clone()
+        };
+        oidc_obj.insert(external_key, Value::Object(provider));
+    }
+
+    oidc_obj
+}
+
 fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8>> {
     let mut root = seed.and_then(parse_object_seed).unwrap_or_default();
 
@@ -261,6 +469,15 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
     root.insert("storageclass".to_string(), Value::Object(sc_obj));
     root.remove("storage_class");
 
+    let oidc_obj = build_oidc_object(cfg);
+    if oidc_obj.is_empty() {
+        root.remove("openid");
+        root.remove(IDENTITY_OPENID_SUB_SYS);
+    } else {
+        root.insert("openid".to_string(), Value::Object(oidc_obj));
+        root.remove(IDENTITY_OPENID_SUB_SYS);
+    }
+
     Ok(serde_json::to_vec(&Value::Object(root))?)
 }
 
@@ -278,6 +495,7 @@ fn is_standard_object_server_config(data: &[u8]) -> bool {
 
 fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool {
     build_storageclass_object(lhs) == build_storageclass_object(rhs)
+        && build_semantic_oidc_object(lhs) == build_semantic_oidc_object(rhs)
 }
 
 fn is_object_not_found(err: &Error) -> bool {
@@ -287,7 +505,14 @@ fn is_object_not_found(err: &Error) -> bool {
 pub async fn try_migrate_server_config<S: StorageAPI>(api: Arc<S>) {
     let config_file = get_config_file();
     match api
-        .get_object_info(RUSTFS_META_BUCKET, &config_file, &ObjectOptions::default())
+        .get_object_info(
+            RUSTFS_META_BUCKET,
+            &config_file,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
         .await
     {
         Ok(_) => {
@@ -360,7 +585,13 @@ pub async fn try_migrate_server_config<S: StorageAPI>(api: Arc<S>) {
 /// Handle the situation where the configuration file does not exist, create and save a new configuration
 async fn handle_missing_config<S: StorageAPI>(api: Arc<S>, context: &str) -> Result<Config> {
     warn!("Configuration not found ({}): Start initializing new configuration", context);
-    let cfg = new_and_save_server_config(api).await?;
+    let cfg = if is_first_cluster_node_local().await {
+        new_and_save_server_config(api.clone()).await?
+    } else {
+        let mut cfg = new_server_config();
+        lookup_configs(&mut cfg, api).await;
+        cfg
+    };
     warn!("Configuration initialization complete ({})", context);
     Ok(cfg)
 }
@@ -375,7 +606,7 @@ pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<C
     let config_file = get_config_file();
 
     // Try to read the configuration file
-    match read_config(api.clone(), &config_file).await {
+    match read_config_no_lock(api.clone(), &config_file).await {
         Ok(data) => read_server_config(api, &data).await,
         Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await,
         Err(err) => handle_config_read_error(err, &config_file),
@@ -389,7 +620,7 @@ async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<C
         warn!("Received empty configuration data, try to reread from '{}'", config_file);
 
         // Try to read the configuration again
-        match read_config(api.clone(), &config_file).await {
+        match read_config_no_lock(api.clone(), &config_file).await {
             Ok(cfg_data) => {
                 // TODO: decrypt
                 let cfg = decode_server_config_blob(&cfg_data)?;
@@ -481,7 +712,9 @@ mod tests {
         configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config,
         storage_class_kvs_mut,
     };
-    use crate::config::Config;
+    use crate::config::{Config, oidc};
+    use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
+    use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
     use serde_json::Value;
 
     #[test]
@@ -523,6 +756,54 @@ mod tests {
         assert_eq!(kvs.get("optimize"), "availability");
     }
 
+    #[test]
+    fn test_decode_server_config_reads_openid_providers() {
+        let input = r#"{
+          "version":"33",
+          "storageclass":{"standard":"EC:2","rrs":"EC:1"},
+          "openid":{
+            "default":{
+              "enable":true,
+              "config_url":"https://example.com/.well-known/openid-configuration",
+              "client_id":"console",
+              "client_secret":"secret-value",
+              "scopes":["openid","profile","email"],
+              "redirect_uri_dynamic":true,
+              "display_name":"Default Provider"
+            },
+            "smoke":{
+              "enable":false,
+              "config_url":"https://issuer.example.com/.well-known/openid-configuration",
+              "client_id":"smoke-client",
+              "scopes":["openid"],
+              "redirect_uri_dynamic":false
+            }
+          }
+        }"#;
+
+        let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
+
+        let default_kvs = cfg
+            .get_value(IDENTITY_OPENID_SUB_SYS, DEFAULT_DELIMITER)
+            .expect("default oidc provider should exist");
+        assert_eq!(
+            default_kvs.get(rustfs_config::oidc::OIDC_CONFIG_URL),
+            "https://example.com/.well-known/openid-configuration"
+        );
+        assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_CLIENT_ID), "console");
+        assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_SCOPES), "openid,profile,email");
+        assert_eq!(default_kvs.get(ENABLE_KEY), EnableState::On.to_string());
+
+        let smoke_kvs = cfg
+            .get_value(IDENTITY_OPENID_SUB_SYS, "smoke")
+            .expect("named oidc provider should exist");
+        assert_eq!(smoke_kvs.get(rustfs_config::oidc::OIDC_CLIENT_ID), "smoke-client");
+        assert_eq!(
+            smoke_kvs.get(rustfs_config::oidc::OIDC_REDIRECT_URI_DYNAMIC),
+            EnableState::Off.to_string()
+        );
+    }
+
     #[test]
     fn test_encode_server_config_writes_external_object_shape() {
         let mut cfg = Config::new();
@@ -537,6 +818,48 @@ mod tests {
         assert!(v.get("storage_class").is_none(), "should not write rustfs map shape");
     }
 
+    #[test]
+    fn test_encode_server_config_writes_openid_object_shape() {
+        let mut cfg = Config::new();
+        let mut oidc_section = std::collections::HashMap::new();
+        let mut default_provider = oidc::DEFAULT_IDENTITY_OPENID_KVS.clone();
+        default_provider.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        default_provider.insert(
+            rustfs_config::oidc::OIDC_CONFIG_URL.to_string(),
+            "https://example.com/.well-known/openid-configuration".to_string(),
+        );
+        default_provider.insert(rustfs_config::oidc::OIDC_CLIENT_ID.to_string(), "console".to_string());
+        default_provider.insert(rustfs_config::oidc::OIDC_SCOPES.to_string(), "openid,profile,email".to_string());
+        oidc_section.insert(DEFAULT_DELIMITER.to_string(), default_provider);
+        cfg.0.insert(IDENTITY_OPENID_SUB_SYS.to_string(), oidc_section);
+
+        let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
+        let v: Value = serde_json::from_slice(&out).expect("output should be json");
+        let openid = v
+            .get("openid")
+            .and_then(Value::as_object)
+            .expect("output should include openid object");
+        let default_provider = openid
+            .get("default")
+            .and_then(Value::as_object)
+            .expect("default provider should be encoded");
+
+        assert_eq!(
+            default_provider
+                .get(rustfs_config::oidc::OIDC_CLIENT_ID)
+                .and_then(Value::as_str),
+            Some("console")
+        );
+        assert_eq!(
+            default_provider
+                .get(rustfs_config::oidc::OIDC_SCOPES)
+                .and_then(Value::as_array)
+                .map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
+            Some(vec!["openid", "profile", "email"])
+        );
+        assert_eq!(default_provider.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
+    }
+
     #[test]
     fn test_is_standard_object_server_config_detection() {
         let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"}}"#;
@@ -554,4 +877,39 @@ mod tests {
         let rhs = decode_server_config_blob(legacy).expect("decode legacy");
         assert!(configs_semantically_equal(&lhs, &rhs));
     }
+
+    #[test]
+    fn test_configs_semantically_equal_accounts_for_openid() {
+        let external = br#"{
+          "version":"33",
+          "storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"},
+          "openid":{
+            "default":{
+              "enable":true,
+              "config_url":"https://example.com/.well-known/openid-configuration",
+              "client_id":"console",
+              "scopes":["openid","profile","email"],
+              "redirect_uri_dynamic":true
+            }
+          }
+        }"#;
+        let legacy = br#"{
+          "storage_class":{"_":[
+            {"key":"standard","value":"EC:2"},
+            {"key":"rrs","value":"EC:1"},
+            {"key":"optimize","value":"availability"}
+          ]},
+          "identity_openid":{"_":[
+            {"key":"enable","value":"on"},
+            {"key":"config_url","value":"https://example.com/.well-known/openid-configuration"},
+            {"key":"client_id","value":"console"},
+            {"key":"scopes","value":"openid,profile,email"},
+            {"key":"redirect_uri_dynamic","value":"on"}
+          ]}
+        }"#;
+
+        let lhs = decode_server_config_blob(external).expect("decode external");
+        let rhs = decode_server_config_blob(legacy).expect("decode legacy");
+        assert!(configs_semantically_equal(&lhs, &rhs));
+    }
 }

crates/ecstore/src/config/mod.rs

@@ -79,7 +79,7 @@ pub async fn try_migrate_server_config(api: Arc<ECStore>) {
     com::try_migrate_server_config(api).await
 }
 
-#[derive(Debug, Deserialize, Serialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
 pub struct KV {
     pub key: String,
     pub value: String,
@@ -87,7 +87,7 @@ pub struct KV {
     pub hidden_if_empty: bool,
 }
 
-#[derive(Debug, Deserialize, Serialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
 pub struct KVS(pub Vec<KV>);
 
 impl Default for KVS {
@@ -163,7 +163,7 @@ impl KVS {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Config(pub HashMap<String, HashMap<String, KVS>>);
 
 impl Default for Config {

crates/ecstore/src/disk/disk_store.rs

@@ -18,6 +18,7 @@ use crate::disk::{
     WalkDirOptions,
     local::{LocalDisk, ScanGuard},
 };
+use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use std::{
@@ -410,7 +411,19 @@ impl LocalDiskWrapper {
     /// Set the disk ID
     pub async fn set_disk_id_internal(&self, id: Option<Uuid>) -> Result<()> {
         let mut disk_id = self.disk_id.write().await;
+        let previous = *disk_id;
         *disk_id = id;
+        drop(disk_id);
+
+        if self.disk.is_local() {
+            let mut disk_id_map = GLOBAL_LOCAL_DISK_ID_MAP.write().await;
+            if let Some(previous_id) = previous {
+                disk_id_map.remove(&previous_id);
+            }
+            if let Some(current_id) = id {
+                disk_id_map.insert(current_id, self.disk.endpoint().to_string());
+            }
+        }
         Ok(())
     }
 

crates/ecstore/src/disk/local.rs

@@ -29,7 +29,6 @@ use crate::disk::{
     os::{check_path_length, is_empty_dir, is_root_disk, rename_all},
 };
 use crate::erasure_coding::bitrot_verify;
-use crate::file_cache::{get_global_file_cache, prefetch_metadata_patterns, read_metadata_cached};
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
 use bytes::Bytes;
 use parking_lot::RwLock as ParkingLotRwLock;
@@ -492,56 +491,6 @@ impl LocalDisk {
         Ok(results.into_iter().map(|(_, path)| path).collect())
     }
 
-    // Optimized metadata reading with caching
-    async fn read_metadata_cached(&self, path: PathBuf) -> Result<Arc<FileMeta>> {
-        read_metadata_cached(path).await
-    }
-
-    // Smart prefetching for related files
-    async fn read_version_with_prefetch(
-        &self,
-        volume: &str,
-        path: &str,
-        version_id: &str,
-        opts: &ReadOptions,
-    ) -> Result<FileInfo> {
-        let file_path = self.get_object_path(volume, path)?;
-
-        // Async prefetch related files, don't block current read
-        if let Some(parent) = file_path.parent() {
-            prefetch_metadata_patterns(parent, &[STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await;
-        }
-
-        // Main read logic
-        let file_dir = self.get_bucket_path(volume)?;
-        let (data, _) = self.read_raw(volume, file_dir, file_path, opts.read_data).await?;
-
-        get_file_info(
-            &data,
-            volume,
-            path,
-            version_id,
-            FileInfoOpts {
-                data: opts.read_data,
-                include_free_versions: false,
-            },
-        )
-        .map_err(|_e| DiskError::Unexpected)
-    }
-
-    // Batch metadata reading for multiple objects
-    async fn read_metadata_batch(&self, requests: Vec<(String, String)>) -> Result<Vec<Option<Arc<FileMeta>>>> {
-        let paths: Vec<PathBuf> = requests
-            .iter()
-            .map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, STORAGE_FORMAT_FILE)))
-            .collect::<Result<Vec<_>>>()?;
-
-        let cache = get_global_file_cache();
-        let results = cache.get_metadata_batch(paths).await;
-
-        Ok(results.into_iter().map(|r| r.ok()).collect())
-    }
-
     // /// Write to the filesystem atomically.
     // /// This is done by first writing to a temporary location and then moving the file.
     // pub(crate) async fn prepare_file_write<'a>(&self, path: &'a PathBuf) -> Result<FileWriter<'a>> {
@@ -865,8 +814,6 @@ impl LocalDisk {
 
         rename_all(tmp_file_path, &file_path, volume_dir).await?;
 
-        // Invalidate cache after successful write
-        get_global_file_cache().invalidate(&file_path).await;
         Ok(())
     }
 
@@ -891,7 +838,9 @@ impl LocalDisk {
         check_path_length(file_path.to_string_lossy().as_ref())?;
 
         self.write_all_internal(&file_path, InternalBuf::Owned(buf), sync, skip_parent)
-            .await
+            .await?;
+
+        Ok(())
     }
     // write_all_internal do write file
     async fn write_all_internal(&self, file_path: &Path, data: InternalBuf<'_>, sync: bool, skip_parent: &Path) -> Result<()> {
@@ -2478,7 +2427,7 @@ impl DiskAPI for LocalDisk {
                 file_path.as_path(),
                 Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str()),
             ]);
-            return rename_all(src_path, dst_path, file_path).await;
+            return rename_all(&src_path, &dst_path, file_path).await;
         }
 
         self.delete_file(&volume_dir, &xl_path, true, false).await
@@ -2601,17 +2550,9 @@ impl DiskAPI for LocalDisk {
 
     #[tracing::instrument(skip(self))]
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
-        // Try to use cached file content reading for better performance, with safe fallback
         let file_path = self.get_object_path(volume, path)?;
-        // let file_path = file_path.join(Path::new(STORAGE_FORMAT_FILE));
-
-        // First, try the cache
-        if let Ok(bytes) = get_global_file_cache().get_file_content(file_path.clone()).await {
-            return Ok(bytes);
-        }
-
-        // Fallback to direct read if cache fails
-        let (data, _) = self.read_metadata_with_dmtime(&file_path).await?;
+        let volume_dir = self.get_bucket_path(volume)?;
+        let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
         Ok(data.into())
     }
 }

crates/ecstore/src/endpoints.rs

@@ -791,14 +791,17 @@ mod test {
             panic!("No non-loop back IP address found for this host");
         }
         let non_loop_back_ip = non_loop_back_i_ps[0];
+        let remote_ip1 = "192.0.2.10";
+        let remote_ip2 = "192.0.2.11";
+        let remote_ip3 = "192.0.2.12";
 
         let case1_endpoint1 = format!("http://{non_loop_back_ip}/d1");
         let case1_endpoint2 = format!("http://{non_loop_back_ip}/d2");
         let args = vec![
             format!("http://{}:10000/d1", non_loop_back_ip),
             format!("http://{}:10000/d2", non_loop_back_ip),
-            "http://example.org:10000/d3".to_string(),
-            "http://example.com:10000/d4".to_string(),
+            format!("http://{remote_ip1}:10000/d3"),
+            format!("http://{remote_ip2}:10000/d4"),
         ];
         let (case1_ur_ls, case1_local_flags) = get_expected_endpoints(args, format!("http://{non_loop_back_ip}:10000/"));
 
@@ -807,26 +810,26 @@ mod test {
         let args = vec![
             format!("http://{}:10000/d1", non_loop_back_ip),
             format!("http://{}:9000/d2", non_loop_back_ip),
-            "http://example.org:10000/d3".to_string(),
-            "http://example.com:10000/d4".to_string(),
+            format!("http://{remote_ip1}:10000/d3"),
+            format!("http://{remote_ip2}:10000/d4"),
         ];
         let (case2_ur_ls, case2_local_flags) = get_expected_endpoints(args, format!("http://{non_loop_back_ip}:10000/"));
 
         let case3_endpoint1 = format!("http://{non_loop_back_ip}/d1");
         let args = vec![
             format!("http://{}:80/d1", non_loop_back_ip),
-            "http://example.org:9000/d2".to_string(),
-            "http://example.com:80/d3".to_string(),
-            "http://example.net:80/d4".to_string(),
+            format!("http://{remote_ip1}:9000/d2"),
+            format!("http://{remote_ip2}:80/d3"),
+            format!("http://{remote_ip3}:80/d4"),
         ];
         let (case3_ur_ls, case3_local_flags) = get_expected_endpoints(args, format!("http://{non_loop_back_ip}:80/"));
 
         let case4_endpoint1 = format!("http://{non_loop_back_ip}/d1");
         let args = vec![
             format!("http://{}:9000/d1", non_loop_back_ip),
-            "http://example.org:9000/d2".to_string(),
-            "http://example.com:9000/d3".to_string(),
-            "http://example.net:9000/d4".to_string(),
+            format!("http://{remote_ip1}:9000/d2"),
+            format!("http://{remote_ip2}:9000/d3"),
+            format!("http://{remote_ip3}:9000/d4"),
         ];
         let (case4_ur_ls, case4_local_flags) = get_expected_endpoints(args, format!("http://{non_loop_back_ip}:9000/"));
 
@@ -844,8 +847,8 @@ mod test {
 
         let case6_endpoint1 = format!("http://{non_loop_back_ip}:9003/d4");
         let args = vec![
-            "http://localhost:9000/d1".to_string(),
-            "http://localhost:9001/d2".to_string(),
+            "http://127.0.0.1:9000/d1".to_string(),
+            "http://127.0.0.1:9001/d2".to_string(),
             "http://127.0.0.1:9002/d3".to_string(),
             case6_endpoint1.clone(),
         ];
@@ -864,8 +867,8 @@ mod test {
             // Erasure Single Drive
             TestCase {
                 num: 2,
-                server_addr: "localhost:9000",
-                args: vec!["http://localhost/d1"],
+                server_addr: "127.0.0.1:9000",
+                args: vec!["http://127.0.0.1/d1"],
                 expected_err: Some(Error::other("use path style endpoint for single node setup")),
                 ..Default::default()
             },
@@ -885,7 +888,7 @@ mod test {
             },
             TestCase {
                 num: 4,
-                server_addr: "localhost:10000",
+                server_addr: "127.0.0.1:10000",
                 args: vec!["/d1"],
                 expected_endpoints: Some(Endpoints(vec![Endpoint {
                     url: must_file_path("/d1"),
@@ -899,12 +902,12 @@ mod test {
             },
             TestCase {
                 num: 5,
-                server_addr: "localhost:9000",
+                server_addr: "127.0.0.1:9000",
                 args: vec![
                     "https://127.0.0.1:9000/d1",
-                    "https://localhost:9001/d1",
-                    "https://example.com/d1",
-                    "https://example.com/d2",
+                    "https://127.0.0.1:9001/d1",
+                    "https://192.0.2.1/d1",
+                    "https://192.0.2.1/d2",
                 ],
                 expected_err: Some(Error::other("same path '/d1' can not be served by different port on same address")),
                 ..Default::default()
@@ -952,35 +955,35 @@ mod test {
                 num: 7,
                 server_addr: "0.0.0.0:9000",
                 args: vec![
-                    "http://localhost/d1",
-                    "http://localhost/d2",
-                    "http://localhost/d3",
-                    "http://localhost/d4",
+                    "http://127.0.0.1/d1",
+                    "http://127.0.0.1/d2",
+                    "http://127.0.0.1/d3",
+                    "http://127.0.0.1/d4",
                 ],
                 expected_endpoints: Some(Endpoints(vec![
                     Endpoint {
-                        url: must_url("http://localhost:9000/d1"),
+                        url: must_url("http://127.0.0.1:9000/d1"),
                         is_local: true,
                         pool_idx: 0,
                         set_idx: 0,
                         disk_idx: 0,
                     },
                     Endpoint {
-                        url: must_url("http://localhost:9000/d2"),
+                        url: must_url("http://127.0.0.1:9000/d2"),
                         is_local: true,
                         pool_idx: 0,
                         set_idx: 0,
                         disk_idx: 0,
                     },
                     Endpoint {
-                        url: must_url("http://localhost:9000/d3"),
+                        url: must_url("http://127.0.0.1:9000/d3"),
                         is_local: true,
                         pool_idx: 0,
                         set_idx: 0,
                         disk_idx: 0,
                     },
                     Endpoint {
-                        url: must_url("http://localhost:9000/d4"),
+                        url: must_url("http://127.0.0.1:9000/d4"),
                         is_local: true,
                         pool_idx: 0,
                         set_idx: 0,
@@ -995,8 +998,8 @@ mod test {
                 num: 8,
                 server_addr: "127.0.0.1:10000",
                 args: vec![
-                    "http://localhost/d1",
-                    "http://localhost/d2",
+                    "http://[::1]/d1",
+                    "http://[::1]/d2",
                     "http://127.0.0.1/d3",
                     "http://127.0.0.1/d4",
                 ],
@@ -1034,8 +1037,8 @@ mod test {
                 args: vec![
                     case1_endpoint1.as_str(),
                     case1_endpoint2.as_str(),
-                    "http://example.org/d3",
-                    "http://example.com/d4",
+                    "http://192.0.2.10/d3",
+                    "http://192.0.2.11/d4",
                 ],
                 expected_endpoints: Some(Endpoints(vec![
                     Endpoint {
@@ -1076,8 +1079,8 @@ mod test {
                 args: vec![
                     case2_endpoint1.as_str(),
                     case2_endpoint2.as_str(),
-                    "http://example.org/d3",
-                    "http://example.com/d4",
+                    "http://192.0.2.10/d3",
+                    "http://192.0.2.11/d4",
                 ],
                 expected_endpoints: Some(Endpoints(vec![
                     Endpoint {
@@ -1117,9 +1120,9 @@ mod test {
                 server_addr: "0.0.0.0:80",
                 args: vec![
                     case3_endpoint1.as_str(),
-                    "http://example.org:9000/d2",
-                    "http://example.com/d3",
-                    "http://example.net/d4",
+                    "http://192.0.2.10:9000/d2",
+                    "http://192.0.2.11/d3",
+                    "http://192.0.2.12/d4",
                 ],
                 expected_endpoints: Some(Endpoints(vec![
                     Endpoint {
@@ -1159,9 +1162,9 @@ mod test {
                 server_addr: "0.0.0.0:9000",
                 args: vec![
                     case4_endpoint1.as_str(),
-                    "http://example.org/d2",
-                    "http://example.com/d3",
-                    "http://example.net/d4",
+                    "http://192.0.2.10/d2",
+                    "http://192.0.2.11/d3",
+                    "http://192.0.2.12/d4",
                 ],
                 expected_endpoints: Some(Endpoints(vec![
                     Endpoint {
@@ -1242,8 +1245,8 @@ mod test {
                 num: 16,
                 server_addr: "0.0.0.0:9003",
                 args: vec![
-                    "http://localhost:9000/d1",
-                    "http://localhost:9001/d2",
+                    "http://127.0.0.1:9000/d1",
+                    "http://127.0.0.1:9001/d2",
                     "http://127.0.0.1:9002/d3",
                     case6_endpoint1.as_str(),
                 ],

crates/ecstore/src/erasure_coding/bitrot.rs

@@ -15,6 +15,7 @@
 use bytes::Bytes;
 use pin_project_lite::pin_project;
 use rustfs_utils::HashAlgorithm;
+use std::io::IoSlice;
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use tracing::error;
 use uuid::Uuid;
@@ -155,23 +156,49 @@ where
                 error!("bitrot writer write hash error: hash is empty");
                 return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "hash is empty"));
             }
-            self.inner.write_all(hash.as_ref()).await?;
+            write_all_vectored(&mut self.inner, hash.as_ref(), buf).await?;
+        } else {
+            self.inner.write_all(buf).await?;
         }
 
-        self.inner.write_all(buf).await?;
-
-        self.inner.flush().await?;
-
         let n = buf.len();
 
         Ok(n)
     }
 
     pub async fn shutdown(&mut self) -> std::io::Result<()> {
+        self.inner.flush().await?;
         self.inner.shutdown().await
     }
 }
 
+async fn write_all_vectored<W>(writer: &mut W, hash: &[u8], data: &[u8]) -> std::io::Result<()>
+where
+    W: AsyncWrite + Unpin,
+{
+    let mut hash_offset = 0;
+    let mut data_offset = 0;
+
+    while hash_offset < hash.len() || data_offset < data.len() {
+        let slices = [IoSlice::new(&hash[hash_offset..]), IoSlice::new(&data[data_offset..])];
+        let written = writer.write_vectored(&slices).await?;
+        if written == 0 {
+            return Err(std::io::Error::new(std::io::ErrorKind::WriteZero, "failed to write hash and data"));
+        }
+
+        let hash_remaining = hash.len() - hash_offset;
+        if written < hash_remaining {
+            hash_offset += written;
+            continue;
+        }
+
+        hash_offset = hash.len();
+        data_offset += written - hash_remaining;
+    }
+
+    Ok(())
+}
+
 pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize {
     if algo != HashAlgorithm::HighwayHash256S && algo != HashAlgorithm::HighwayHash256SLegacy {
         return size;
@@ -292,6 +319,33 @@ impl AsyncWrite for CustomWriter {
             }
         }
     }
+
+    fn poll_write_vectored(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        bufs: &[IoSlice<'_>],
+    ) -> std::task::Poll<std::io::Result<usize>> {
+        match self.get_mut() {
+            Self::InlineBuffer(data) => {
+                let total = bufs.iter().map(|buf| buf.len()).sum::<usize>();
+                for buf in bufs {
+                    data.extend_from_slice(buf);
+                }
+                std::task::Poll::Ready(Ok(total))
+            }
+            Self::Other(writer) => {
+                let pinned_writer = std::pin::Pin::new(writer.as_mut());
+                pinned_writer.poll_write_vectored(cx, bufs)
+            }
+        }
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        match self {
+            Self::InlineBuffer(_) => true,
+            Self::Other(writer) => writer.is_write_vectored(),
+        }
+    }
 }
 
 /// Wrapper around BitrotWriter that uses our custom writer
@@ -361,7 +415,74 @@ mod tests {
     use super::BitrotReader;
     use super::BitrotWriter;
     use rustfs_utils::HashAlgorithm;
-    use std::io::Cursor;
+    use std::io::{Cursor, IoSlice};
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+    use std::task::{Context, Poll};
+    use tokio::io::AsyncWrite;
+
+    #[derive(Default)]
+    struct VectoredCountingWriter {
+        vectored_writes: Arc<AtomicUsize>,
+        writes: Vec<u8>,
+    }
+
+    impl AsyncWrite for VectoredCountingWriter {
+        fn poll_write(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            Poll::Ready(Err(std::io::Error::other("poll_write should not be used")))
+        }
+
+        fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_write_vectored(
+            mut self: std::pin::Pin<&mut Self>,
+            _cx: &mut Context<'_>,
+            bufs: &[IoSlice<'_>],
+        ) -> Poll<std::io::Result<usize>> {
+            self.vectored_writes.fetch_add(1, Ordering::SeqCst);
+            let total = bufs.iter().map(|buf| buf.len()).sum::<usize>();
+            for buf in bufs {
+                self.writes.extend_from_slice(buf);
+            }
+            Poll::Ready(Ok(total))
+        }
+
+        fn is_write_vectored(&self) -> bool {
+            true
+        }
+    }
+
+    #[derive(Default)]
+    struct CountingWriter {
+        flushes: Arc<AtomicUsize>,
+        shutdowns: Arc<AtomicUsize>,
+        writes: Vec<u8>,
+    }
+
+    impl AsyncWrite for CountingWriter {
+        fn poll_write(mut self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            self.writes.extend_from_slice(buf);
+            Poll::Ready(Ok(buf.len()))
+        }
+
+        fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            self.flushes.fetch_add(1, Ordering::SeqCst);
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            self.shutdowns.fetch_add(1, Ordering::SeqCst);
+            Poll::Ready(Ok(()))
+        }
+    }
 
     #[tokio::test]
     async fn test_bitrot_read_write_ok() {
@@ -471,4 +592,41 @@ mod tests {
         assert_eq!(n, data_size);
         assert_eq!(data, &out[..]);
     }
+
+    #[tokio::test]
+    async fn test_bitrot_writer_flushes_once_on_shutdown() {
+        let flushes = Arc::new(AtomicUsize::new(0));
+        let shutdowns = Arc::new(AtomicUsize::new(0));
+        let writer = CountingWriter {
+            flushes: flushes.clone(),
+            shutdowns: shutdowns.clone(),
+            writes: Vec::new(),
+        };
+        let mut bitrot_writer = BitrotWriter::new(writer, 8, HashAlgorithm::None);
+
+        bitrot_writer.write(b"12345678").await.unwrap();
+        bitrot_writer.write(b"abc").await.unwrap();
+
+        assert_eq!(flushes.load(Ordering::SeqCst), 0);
+        assert_eq!(shutdowns.load(Ordering::SeqCst), 0);
+
+        bitrot_writer.shutdown().await.unwrap();
+
+        assert_eq!(flushes.load(Ordering::SeqCst), 1);
+        assert_eq!(shutdowns.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn test_bitrot_writer_uses_vectored_write_for_hash_and_data() {
+        let vectored_writes = Arc::new(AtomicUsize::new(0));
+        let writer = VectoredCountingWriter {
+            vectored_writes: vectored_writes.clone(),
+            writes: Vec::new(),
+        };
+        let mut bitrot_writer = BitrotWriter::new(writer, 8, HashAlgorithm::HighwayHash256);
+
+        bitrot_writer.write(b"payload").await.unwrap();
+
+        assert!(vectored_writes.load(Ordering::SeqCst) > 0);
+    }
 }

crates/ecstore/src/erasure_coding/encode.rs

@@ -112,11 +112,66 @@ impl<'a> MultiWriter<'a> {
         )))
     }
 
-    pub async fn _shutdown(&mut self) -> std::io::Result<()> {
-        for writer in self.writers.iter_mut().flatten() {
-            writer.shutdown().await?;
+    async fn shutdown_writer(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>) {
+        match writer_opt {
+            Some(writer) => match writer.shutdown().await {
+                Ok(()) => {
+                    *err = None;
+                }
+                Err(e) => {
+                    *err = Some(Error::from(e));
+                    *writer_opt = None;
+                }
+            },
+            None => {
+                *err = Some(Error::DiskNotFound);
+            }
         }
-        Ok(())
+    }
+
+    pub async fn shutdown(&mut self) -> std::io::Result<()> {
+        {
+            let mut futures = FuturesUnordered::new();
+            for (writer_opt, err) in self.writers.iter_mut().zip(self.errs.iter_mut()) {
+                if err.is_some() {
+                    continue;
+                }
+                futures.push(Self::shutdown_writer(writer_opt, err));
+            }
+            while let Some(()) = futures.next().await {}
+        }
+
+        let nil_count = self.errs.iter().filter(|&e| e.is_none()).count();
+        if nil_count >= self.write_quorum {
+            return Ok(());
+        }
+
+        if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
+            error!(
+                "reduce_write_quorum_errs during shutdown: {:?}, offline-disks={}/{}, errs={:?}",
+                write_err,
+                count_errs(&self.errs, &Error::DiskNotFound),
+                self.writers.len(),
+                self.errs
+            );
+            return Err(std::io::Error::other(format!(
+                "Failed to shutdown writers: {} (offline-disks={}/{})",
+                write_err,
+                count_errs(&self.errs, &Error::DiskNotFound),
+                self.writers.len()
+            )));
+        }
+
+        Err(std::io::Error::other(format!(
+            "Failed to shutdown writers: (offline-disks={}/{}): {}",
+            count_errs(&self.errs, &Error::DiskNotFound),
+            self.writers.len(),
+            self.errs
+                .iter()
+                .map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
+                .collect::<Vec<_>>()
+                .join(", ")
+        )))
     }
 }
 
@@ -176,7 +231,69 @@ impl Erasure {
         }
 
         let (reader, total) = task.await??;
-        // writers.shutdown().await?;
+        writers.shutdown().await?;
         Ok((reader, total))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::erasure_coding::{BitrotWriterWrapper, CustomWriter};
+    use rustfs_utils::HashAlgorithm;
+    use std::pin::Pin;
+    use std::sync::{Arc, Mutex};
+    use std::task::{Context, Poll};
+    use tokio::io::AsyncWrite;
+
+    #[derive(Clone, Default)]
+    struct DeferredCommitWriter {
+        buffered: Vec<u8>,
+        committed: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl DeferredCommitWriter {
+        fn new(committed: Arc<Mutex<Vec<u8>>>) -> Self {
+            Self {
+                buffered: Vec::new(),
+                committed,
+            }
+        }
+    }
+
+    impl AsyncWrite for DeferredCommitWriter {
+        fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            self.buffered.extend_from_slice(buf);
+            Poll::Ready(Ok(buf.len()))
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            let buffered = std::mem::take(&mut self.buffered);
+            let mut committed = self.committed.lock().unwrap();
+            committed.extend_from_slice(&buffered);
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[tokio::test]
+    async fn encode_shutdowns_writers_after_small_shards() {
+        let committed = Arc::new(Mutex::new(Vec::new()));
+        let writer = DeferredCommitWriter::new(committed.clone());
+        let mut writers = vec![Some(BitrotWriterWrapper::new(
+            CustomWriter::new_tokio_writer(writer),
+            16,
+            HashAlgorithm::HighwayHash256S,
+        ))];
+
+        let erasure = Arc::new(Erasure::new(1, 0, 16));
+        let reader = tokio::io::BufReader::new(std::io::Cursor::new(b"small payload".to_vec()));
+        let (_reader, written) = erasure.encode(reader, &mut writers, 1).await.unwrap();
+
+        assert_eq!(written, b"small payload".len());
+        assert!(!committed.lock().unwrap().is_empty());
+    }
+}

crates/ecstore/src/file_cache.rs

@@ -1,332 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! High-performance file content and metadata caching using moka
-//!
-//! This module provides optimized caching for file operations to reduce
-//! redundant I/O and improve overall system performance.
-
-use super::disk::error::{Error, Result};
-use bytes::Bytes;
-use moka::future::Cache;
-use rustfs_filemeta::FileMeta;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use std::time::Duration;
-
-pub struct OptimizedFileCache {
-    // Use moka as high-performance async cache
-    metadata_cache: Cache<PathBuf, Arc<FileMeta>>,
-    file_content_cache: Cache<PathBuf, Bytes>,
-    // Performance monitoring
-    cache_hits: std::sync::atomic::AtomicU64,
-    cache_misses: std::sync::atomic::AtomicU64,
-}
-
-impl OptimizedFileCache {
-    pub fn new() -> Self {
-        Self {
-            metadata_cache: Cache::builder()
-                .max_capacity(2048)
-                .time_to_live(Duration::from_secs(300)) // 5 minutes TTL
-                .time_to_idle(Duration::from_secs(60)) // 1 minute idle
-                .build(),
-
-            file_content_cache: Cache::builder()
-                .max_capacity(512) // Smaller file content cache
-                .time_to_live(Duration::from_secs(120))
-                .weigher(|_key: &PathBuf, value: &Bytes| value.len() as u32)
-                .build(),
-
-            cache_hits: std::sync::atomic::AtomicU64::new(0),
-            cache_misses: std::sync::atomic::AtomicU64::new(0),
-        }
-    }
-
-    pub async fn get_metadata(&self, path: PathBuf) -> Result<Arc<FileMeta>> {
-        if let Some(cached) = self.metadata_cache.get(&path).await {
-            self.cache_hits.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-            return Ok(cached);
-        }
-
-        self.cache_misses.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-
-        // Cache miss, read file
-        let data = tokio::fs::read(&path)
-            .await
-            .map_err(|e| Error::other(format!("Read metadata failed: {e}")))?;
-
-        let mut meta = FileMeta::default();
-        meta.unmarshal_msg(&data)?;
-
-        let arc_meta = Arc::new(meta);
-        self.metadata_cache.insert(path, arc_meta.clone()).await;
-
-        Ok(arc_meta)
-    }
-
-    pub async fn get_file_content(&self, path: PathBuf) -> Result<Bytes> {
-        if let Some(cached) = self.file_content_cache.get(&path).await {
-            self.cache_hits.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-            return Ok(cached);
-        }
-
-        self.cache_misses.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-
-        let data = tokio::fs::read(&path)
-            .await
-            .map_err(|e| Error::other(format!("Read file failed: {e}")))?;
-
-        let bytes = Bytes::from(data);
-        self.file_content_cache.insert(path, bytes.clone()).await;
-
-        Ok(bytes)
-    }
-
-    // Prefetch related files
-    pub async fn prefetch_related(&self, base_path: &Path, patterns: &[&str]) {
-        let mut prefetch_tasks = Vec::new();
-
-        for pattern in patterns {
-            let path = base_path.join(pattern);
-            if tokio::fs::metadata(&path).await.is_ok() {
-                let cache = self.clone();
-                let path_clone = path.clone();
-                prefetch_tasks.push(async move {
-                    let _ = cache.get_metadata(path_clone).await;
-                });
-            }
-        }
-
-        // Parallel prefetch, don't wait for completion
-        if !prefetch_tasks.is_empty() {
-            tokio::spawn(async move {
-                futures::future::join_all(prefetch_tasks).await;
-            });
-        }
-    }
-
-    // Batch metadata reading with deduplication
-    pub async fn get_metadata_batch(
-        &self,
-        paths: Vec<PathBuf>,
-    ) -> Vec<std::result::Result<Arc<FileMeta>, rustfs_filemeta::Error>> {
-        let mut results = Vec::with_capacity(paths.len());
-        let mut cache_futures = Vec::new();
-
-        // First, attempt to get from cache
-        for (i, path) in paths.iter().enumerate() {
-            if let Some(cached) = self.metadata_cache.get(path).await {
-                results.push((i, Ok(cached)));
-                self.cache_hits.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-            } else {
-                cache_futures.push((i, path.clone()));
-            }
-        }
-
-        // For cache misses, read from filesystem
-        if !cache_futures.is_empty() {
-            let mut fs_results = Vec::new();
-
-            for (i, path) in cache_futures {
-                self.cache_misses.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-
-                match tokio::fs::read(&path).await {
-                    Ok(data) => {
-                        let mut meta = FileMeta::default();
-                        match meta.unmarshal_msg(&data) {
-                            Ok(_) => {
-                                let arc_meta = Arc::new(meta);
-                                self.metadata_cache.insert(path, arc_meta.clone()).await;
-                                fs_results.push((i, Ok(arc_meta)));
-                            }
-                            Err(e) => {
-                                fs_results.push((i, Err(e)));
-                            }
-                        }
-                    }
-                    Err(_e) => {
-                        fs_results.push((i, Err(rustfs_filemeta::Error::Unexpected)));
-                    }
-                }
-            }
-
-            results.extend(fs_results);
-        }
-
-        // Sort results back to original order
-        results.sort_by_key(|(i, _)| *i);
-        results.into_iter().map(|(_, result)| result).collect()
-    }
-
-    // Invalidate cache entries for a path
-    pub async fn invalidate(&self, path: &Path) {
-        self.metadata_cache.remove(path).await;
-        self.file_content_cache.remove(path).await;
-    }
-
-    // Get cache statistics
-    pub fn get_stats(&self) -> FileCacheStats {
-        let hits = self.cache_hits.load(std::sync::atomic::Ordering::Relaxed);
-        let misses = self.cache_misses.load(std::sync::atomic::Ordering::Relaxed);
-        let hit_rate = if hits + misses > 0 {
-            (hits as f64 / (hits + misses) as f64) * 100.0
-        } else {
-            0.0
-        };
-
-        FileCacheStats {
-            metadata_cache_size: self.metadata_cache.entry_count(),
-            content_cache_size: self.file_content_cache.entry_count(),
-            cache_hits: hits,
-            cache_misses: misses,
-            hit_rate,
-            total_weight: 0, // Simplified for compatibility
-        }
-    }
-
-    // Clear all caches
-    pub async fn clear(&self) {
-        self.metadata_cache.invalidate_all();
-        self.file_content_cache.invalidate_all();
-
-        // Wait for invalidation to complete
-        self.metadata_cache.run_pending_tasks().await;
-        self.file_content_cache.run_pending_tasks().await;
-    }
-}
-
-impl Clone for OptimizedFileCache {
-    fn clone(&self) -> Self {
-        Self {
-            metadata_cache: self.metadata_cache.clone(),
-            file_content_cache: self.file_content_cache.clone(),
-            cache_hits: std::sync::atomic::AtomicU64::new(self.cache_hits.load(std::sync::atomic::Ordering::Relaxed)),
-            cache_misses: std::sync::atomic::AtomicU64::new(self.cache_misses.load(std::sync::atomic::Ordering::Relaxed)),
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct FileCacheStats {
-    pub metadata_cache_size: u64,
-    pub content_cache_size: u64,
-    pub cache_hits: u64,
-    pub cache_misses: u64,
-    pub hit_rate: f64,
-    pub total_weight: u64,
-}
-
-impl Default for OptimizedFileCache {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-// Global cache instance
-use std::sync::OnceLock;
-
-static GLOBAL_FILE_CACHE: OnceLock<OptimizedFileCache> = OnceLock::new();
-
-pub fn get_global_file_cache() -> &'static OptimizedFileCache {
-    GLOBAL_FILE_CACHE.get_or_init(OptimizedFileCache::new)
-}
-
-// Utility functions for common operations
-pub async fn read_metadata_cached(path: PathBuf) -> Result<Arc<FileMeta>> {
-    get_global_file_cache().get_metadata(path).await
-}
-
-pub async fn read_file_content_cached(path: PathBuf) -> Result<Bytes> {
-    get_global_file_cache().get_file_content(path).await
-}
-
-pub async fn prefetch_metadata_patterns(base_path: &Path, patterns: &[&str]) {
-    get_global_file_cache().prefetch_related(base_path, patterns).await;
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::io::Write;
-    use tempfile::tempdir;
-
-    #[tokio::test]
-    async fn test_file_cache_basic() {
-        let cache = OptimizedFileCache::new();
-
-        // Create a temporary file
-        let dir = tempdir().unwrap();
-        let file_path = dir.path().join("test.txt");
-        let mut file = std::fs::File::create(&file_path).unwrap();
-        writeln!(file, "test content").unwrap();
-        drop(file);
-
-        // First read should be cache miss
-        let content1 = cache.get_file_content(file_path.clone()).await.unwrap();
-        assert_eq!(content1, Bytes::from("test content\n"));
-
-        // Second read should be cache hit
-        let content2 = cache.get_file_content(file_path.clone()).await.unwrap();
-        assert_eq!(content2, content1);
-
-        let stats = cache.get_stats();
-        assert!(stats.cache_hits > 0);
-        assert!(stats.cache_misses > 0);
-    }
-
-    #[tokio::test]
-    async fn test_metadata_batch_read() {
-        let cache = OptimizedFileCache::new();
-
-        // Create test files
-        let dir = tempdir().unwrap();
-        let mut paths = Vec::new();
-
-        for i in 0..5 {
-            let file_path = dir.path().join(format!("test_{i}.txt"));
-            let mut file = std::fs::File::create(&file_path).unwrap();
-            writeln!(file, "content {i}").unwrap();
-            paths.push(file_path);
-        }
-
-        // Note: This test would need actual FileMeta files to work properly
-        // For now, we just test that the function runs without errors
-        let results = cache.get_metadata_batch(paths).await;
-        assert_eq!(results.len(), 5);
-    }
-
-    #[tokio::test]
-    async fn test_cache_invalidation() {
-        let cache = OptimizedFileCache::new();
-
-        let dir = tempdir().unwrap();
-        let file_path = dir.path().join("test.txt");
-        let mut file = std::fs::File::create(&file_path).unwrap();
-        writeln!(file, "test content").unwrap();
-        drop(file);
-
-        // Read file to populate cache
-        let _ = cache.get_file_content(file_path.clone()).await.unwrap();
-
-        // Invalidate cache
-        cache.invalidate(&file_path).await;
-
-        // Next read should be cache miss again
-        let _ = cache.get_file_content(file_path.clone()).await.unwrap();
-
-        let stats = cache.get_stats();
-        assert!(stats.cache_misses >= 2);
-    }
-}

crates/ecstore/src/global.rs

@@ -46,6 +46,7 @@ lazy_static! {
     pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
     pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
     pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
+    pub static ref GLOBAL_LOCAL_DISK_ID_MAP: Arc<RwLock<HashMap<Uuid, String>>> = Arc::new(RwLock::new(HashMap::new()));
     pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
     pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new();
     pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
@@ -153,6 +154,10 @@ pub fn get_global_endpoints_opt() -> Option<EndpointServerPools> {
     GLOBAL_Endpoints.get().cloned()
 }
 
+pub async fn is_first_cluster_node_local() -> bool {
+    get_global_endpoints().first_local()
+}
+
 pub fn get_global_tier_config_mgr() -> Arc<RwLock<TierConfigMgr>> {
     GLOBAL_TierConfigMgr.clone()
 }

crates/ecstore/src/lib.rs

@@ -28,7 +28,6 @@ pub mod disks_layout;
 pub mod endpoints;
 pub mod erasure_coding;
 pub mod error;
-pub mod file_cache;
 pub mod global;
 pub mod metrics_realtime;
 pub mod notification_sys;

crates/ecstore/src/metrics_realtime.rs

@@ -14,8 +14,11 @@
 
 use crate::{admin_server_info::get_local_server_property, new_object_layer_fn, store_api::StorageAPI};
 use chrono::Utc;
-use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, metrics::global_metrics};
-use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, RealtimeMetrics};
+use rustfs_common::{
+    GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, internode_metrics::global_internode_metrics,
+    metrics::global_metrics,
+};
+use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics};
 use rustfs_utils::os::get_drive_stats;
 use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
@@ -120,13 +123,50 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
 
     // if types.contains(&MetricType::SITE_RESYNC) {}
 
-    // if types.contains(&MetricType::NET) {}
+    if types.contains(&MetricType::NET) {
+        let snapshot = global_internode_metrics().snapshot();
+        real_time_metrics.aggregated.net = Some(NetMetrics {
+            collected_at: Utc::now(),
+            interface_name: "internode".to_string(),
+            net_stats: NetDevLine {
+                name: "internode".to_string(),
+                rx_bytes: snapshot.recv_bytes_total,
+                tx_bytes: snapshot.sent_bytes_total,
+                ..Default::default()
+            },
+        });
+    }
 
     // if types.contains(&MetricType::MEM) {}
 
     // if types.contains(&MetricType::CPU) {}
 
-    // if types.contains(&MetricType::RPC) {}
+    if types.contains(&MetricType::RPC) {
+        let collected_at = Utc::now();
+        let snapshot = global_internode_metrics().snapshot();
+        let last_connect_time =
+            chrono::DateTime::<Utc>::from_timestamp_millis(snapshot.last_dial_unix_millis as i64).unwrap_or(collected_at);
+
+        real_time_metrics.aggregated.rpc = Some(RPCMetrics {
+            collected_at,
+            connected: i32::from(snapshot.last_dial_unix_millis > 0),
+            reconnect_count: snapshot.dial_errors_total.min(i32::MAX as u64) as i32,
+            disconnected: 0,
+            outgoing_streams: 0,
+            incoming_streams: 0,
+            outgoing_bytes: snapshot.sent_bytes_total.min(i64::MAX as u64) as i64,
+            incoming_bytes: snapshot.recv_bytes_total.min(i64::MAX as u64) as i64,
+            outgoing_messages: snapshot.outgoing_requests_total.min(i64::MAX as u64) as i64,
+            incoming_messages: snapshot.incoming_requests_total.min(i64::MAX as u64) as i64,
+            out_queue: 0,
+            last_pong_time: collected_at,
+            last_ping_ms: snapshot.dial_avg_time_nanos as f64 / 1_000_000.0,
+            max_ping_dur_ms: snapshot.dial_avg_time_nanos as f64 / 1_000_000.0,
+            last_connect_time,
+            by_destination: None,
+            by_caller: None,
+        });
+    }
 
     real_time_metrics
         .by_host
@@ -211,7 +251,9 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
 
 #[cfg(test)]
 mod test {
-    use super::MetricType;
+    use super::*;
+    use rustfs_common::internode_metrics::global_internode_metrics;
+    use std::time::Duration;
 
     #[test]
     fn tes_types() {
@@ -229,4 +271,30 @@ mod test {
         let disk = MetricType::new(1 << 1);
         assert!(disk.contains(&MetricType::DISK));
     }
+
+    #[tokio::test]
+    async fn collect_local_metrics_reports_internode_net_and_rpc() {
+        let metrics = global_internode_metrics();
+        metrics.reset_for_test();
+        metrics.record_sent_bytes(128);
+        metrics.record_recv_bytes(64);
+        metrics.record_outgoing_request();
+        metrics.record_incoming_request();
+        metrics.record_dial_result(Duration::from_millis(4), true);
+
+        let realtime = collect_local_metrics(MetricType::NET, &CollectMetricsOpts::default()).await;
+        let net = realtime.aggregated.net.expect("net metrics");
+        assert_eq!(net.net_stats.tx_bytes, 128);
+        assert_eq!(net.net_stats.rx_bytes, 64);
+
+        let realtime = collect_local_metrics(MetricType::RPC, &CollectMetricsOpts::default()).await;
+        let rpc = realtime.aggregated.rpc.expect("rpc metrics");
+        assert_eq!(rpc.outgoing_bytes, 128);
+        assert_eq!(rpc.incoming_bytes, 64);
+        assert_eq!(rpc.outgoing_messages, 1);
+        assert_eq!(rpc.incoming_messages, 1);
+        assert!(rpc.last_ping_ms > 0.0);
+
+        metrics.reset_for_test();
+    }
 }

crates/ecstore/src/rpc/remote_disk.rs

@@ -23,6 +23,7 @@ use crate::disk::{
 };
 use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
 use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::set_disk::DEFAULT_READ_BUFFER_SIZE;
 use crate::{
     disk::error::{Error, Result},
     rpc::build_auth_headers,
@@ -40,7 +41,9 @@ use rustfs_protos::proto_gen::node_service::{
     node_service_client::NodeServiceClient,
 };
 use rustfs_rio::{HttpReader, HttpWriter};
+use serde::{Serialize, de::DeserializeOwned};
 use std::{
+    io::Cursor,
     path::PathBuf,
     sync::{
         Arc,
@@ -49,12 +52,36 @@ use std::{
     time::Duration,
 };
 use tokio::time;
-use tokio::{io::AsyncWrite, net::TcpStream, time::timeout};
+use tokio::{
+    io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt},
+    net::TcpStream,
+    time::timeout,
+};
 use tokio_util::sync::CancellationToken;
 use tonic::{Request, service::interceptor::InterceptedService, transport::Channel};
 use tracing::{debug, info, warn};
 use uuid::Uuid;
 
+async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
+where
+    R: AsyncRead + Unpin,
+    W: AsyncWrite + Unpin,
+{
+    let mut copied = 0_u64;
+    let mut buffer = vec![0_u8; buffer_size];
+
+    loop {
+        let bytes_read = reader.read(&mut buffer).await?;
+        if bytes_read == 0 {
+            writer.flush().await?;
+            return Ok(copied);
+        }
+
+        writer.write_all(&buffer[..bytes_read]).await?;
+        copied += bytes_read as u64;
+    }
+}
+
 #[derive(Debug)]
 pub struct RemoteDisk {
     pub id: Mutex<Option<Uuid>>,
@@ -259,6 +286,27 @@ impl RemoteDisk {
             .await
             .map_err(|err| Error::other(format!("can not get client, err: {err}")))
     }
+
+    async fn disk_ref(&self) -> String {
+        (*self.id.lock().await)
+            .map(|id| id.to_string())
+            .unwrap_or_else(|| self.endpoint.to_string())
+    }
+}
+
+fn encode_msgpack<T: Serialize>(value: &T) -> Result<Vec<u8>> {
+    let mut serializer = rmp_serde::Serializer::new(Vec::new());
+    value.serialize(&mut serializer)?;
+    Ok(serializer.into_inner())
+}
+
+fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str) -> Result<T> {
+    if !binary.is_empty() {
+        let mut deserializer = rmp_serde::Deserializer::new(Cursor::new(binary));
+        return T::deserialize(&mut deserializer).map_err(Error::from);
+    }
+
+    serde_json::from_str(json).map_err(Error::from)
 }
 
 // TODO: all api need to handle errors
@@ -707,18 +755,21 @@ impl DiskAPI for RemoteDisk {
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
         info!("write_metadata {}/{}", volume, path);
         let file_info = serde_json::to_string(&fi)?;
+        let file_info_bin = encode_msgpack(&fi)?;
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(WriteMetadataRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     file_info: file_info.clone(),
+                    file_info_bin: file_info_bin.clone(),
                 });
 
                 let response = client.write_metadata(request).await?.into_inner();
@@ -735,6 +786,7 @@ impl DiskAPI for RemoteDisk {
     }
 
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
+        let disk = self.disk_ref().await;
         let mut client = self
             .get_client()
             .await
@@ -742,7 +794,7 @@ impl DiskAPI for RemoteDisk {
         let request = Request::new(ReadMetadataRequest {
             volume: volume.to_string(),
             path: path.to_string(),
-            disk: self.endpoint.to_string(),
+            disk,
         });
 
         let response = client.read_metadata(request).await?.into_inner();
@@ -759,19 +811,24 @@ impl DiskAPI for RemoteDisk {
         info!("update_metadata");
         let file_info = serde_json::to_string(&fi)?;
         let opts_str = serde_json::to_string(&opts)?;
+        let file_info_bin = encode_msgpack(&fi)?;
+        let opts_bin = encode_msgpack(opts)?;
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(UpdateMetadataRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     file_info: file_info.clone(),
                     opts: opts_str.clone(),
+                    file_info_bin: file_info_bin.clone(),
+                    opts_bin: opts_bin.clone(),
                 });
 
                 let response = client.update_metadata(request).await?.into_inner();
@@ -798,19 +855,22 @@ impl DiskAPI for RemoteDisk {
     ) -> Result<FileInfo> {
         info!("read_version");
         let opts_str = serde_json::to_string(opts)?;
+        let opts_bin = encode_msgpack(opts)?;
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(ReadVersionRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     version_id: version_id.to_string(),
                     opts: opts_str.clone(),
+                    opts_bin: opts_bin.clone(),
                 });
 
                 let response = client.read_version(request).await?.into_inner();
@@ -819,7 +879,7 @@ impl DiskAPI for RemoteDisk {
                     return Err(response.error.unwrap_or_default().into());
                 }
 
-                let file_info = serde_json::from_str::<FileInfo>(&response.file_info)?;
+                let file_info = decode_msgpack_or_json::<FileInfo>(&response.file_info_bin, &response.file_info)?;
 
                 Ok(file_info)
             },
@@ -834,12 +894,13 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(ReadXlRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     read_data,
@@ -851,7 +912,7 @@ impl DiskAPI for RemoteDisk {
                     return Err(response.error.unwrap_or_default().into());
                 }
 
-                let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
+                let raw_file_info = decode_msgpack_or_json::<RawFileInfo>(&response.raw_file_info_bin, &response.raw_file_info)?;
 
                 Ok(raw_file_info)
             },
@@ -909,13 +970,14 @@ impl DiskAPI for RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
+        let disk = self.disk_ref().await;
 
         let mut client = self
             .get_client()
             .await
             .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
         let request = Request::new(ListDirRequest {
-            disk: self.endpoint.to_string(),
+            disk,
             volume: volume.to_string(),
             dir_path: dir_path.to_string(),
             count,
@@ -937,12 +999,9 @@ impl DiskAPI for RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
+        let disk = self.disk_ref().await;
 
-        let url = format!(
-            "{}/rustfs/rpc/walk_dir?disk={}",
-            self.endpoint.grid_host(),
-            urlencoding::encode(self.endpoint.to_string().as_str()),
-        );
+        let url = format!("{}/rustfs/rpc/walk_dir?disk={}", self.endpoint.grid_host(), urlencoding::encode(&disk),);
 
         let opts = serde_json::to_vec(&opts)?;
 
@@ -952,33 +1011,14 @@ impl DiskAPI for RemoteDisk {
 
         let mut reader = HttpReader::new(url, Method::GET, headers, Some(opts)).await?;
 
-        tokio::io::copy(&mut reader, wr).await?;
+        copy_stream_with_buffer(&mut reader, wr, DEFAULT_READ_BUFFER_SIZE).await?;
 
         Ok(())
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
     async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
-        info!("read_file {}/{}", volume, path);
-
-        if self.health.is_faulty() {
-            return Err(DiskError::FaultyDisk);
-        }
-
-        let url = format!(
-            "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
-            self.endpoint.grid_host(),
-            urlencoding::encode(self.endpoint.to_string().as_str()),
-            urlencoding::encode(volume),
-            urlencoding::encode(path),
-            0,
-            0
-        );
-
-        let mut headers = HeaderMap::new();
-        headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
-        build_auth_headers(&url, &Method::GET, &mut headers);
-        Ok(Box::new(HttpReader::new(url, Method::GET, headers, None).await?))
+        self.read_file_stream(volume, path, 0, 0).await
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
@@ -995,11 +1035,12 @@ impl DiskAPI for RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
+        let disk = self.disk_ref().await;
 
         let url = format!(
             "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
             self.endpoint.grid_host(),
-            urlencoding::encode(self.endpoint.to_string().as_str()),
+            urlencoding::encode(&disk),
             urlencoding::encode(volume),
             urlencoding::encode(path),
             offset,
@@ -1019,11 +1060,12 @@ impl DiskAPI for RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
+        let disk = self.disk_ref().await;
 
         let url = format!(
             "{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
             self.endpoint.grid_host(),
-            urlencoding::encode(self.endpoint.to_string().as_str()),
+            urlencoding::encode(&disk),
             urlencoding::encode(volume),
             urlencoding::encode(path),
             true,
@@ -1049,11 +1091,12 @@ impl DiskAPI for RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
+        let disk = self.disk_ref().await;
 
         let url = format!(
             "{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
             self.endpoint.grid_host(),
-            urlencoding::encode(self.endpoint.to_string().as_str()),
+            urlencoding::encode(&disk),
             urlencoding::encode(volume),
             urlencoding::encode(path),
             false,
@@ -1261,13 +1304,16 @@ impl DiskAPI for RemoteDisk {
         self.execute_with_timeout(
             || async {
                 let read_multiple_req = serde_json::to_string(&req)?;
+                let read_multiple_req_bin = encode_msgpack(&req)?;
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(ReadMultipleRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     read_multiple_req,
+                    read_multiple_req_bin,
                 });
 
                 let response = client.read_multiple(request).await?.into_inner();
@@ -1276,11 +1322,19 @@ impl DiskAPI for RemoteDisk {
                     return Err(response.error.unwrap_or_default().into());
                 }
 
-                let read_multiple_resps = response
-                    .read_multiple_resps
-                    .into_iter()
-                    .filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
-                    .collect();
+                let read_multiple_resps = if !response.read_multiple_resps_bin.is_empty() {
+                    response
+                        .read_multiple_resps_bin
+                        .into_iter()
+                        .filter_map(|buf| decode_msgpack_or_json::<ReadMultipleResp>(&buf, "").ok())
+                        .collect()
+                } else {
+                    response
+                        .read_multiple_resps
+                        .into_iter()
+                        .filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
+                        .collect()
+                };
 
                 Ok(read_multiple_resps)
             },
@@ -1295,12 +1349,13 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(WriteAllRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     data,
@@ -1325,12 +1380,13 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(ReadAllRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                 });
@@ -1386,6 +1442,7 @@ impl DiskAPI for RemoteDisk {
 mod tests {
     use super::*;
     use std::sync::Once;
+    use tokio::io::duplex;
     use tokio::net::TcpListener;
     use tracing::Level;
     use uuid::Uuid;
@@ -1543,6 +1600,24 @@ mod tests {
         assert!(!remote_disk.is_online().await);
     }
 
+    #[tokio::test]
+    async fn test_copy_stream_with_buffer_copies_full_payload() {
+        let payload = b"walk-dir-stream".repeat(1024);
+        let expected = payload.clone();
+        let (mut write_half, mut read_half) = duplex(128);
+
+        let copy_task = tokio::spawn(async move {
+            let mut cursor = Cursor::new(payload);
+            copy_stream_with_buffer(&mut cursor, &mut write_half, 4 * 1024).await.unwrap();
+        });
+
+        let mut copied = Vec::new();
+        read_half.read_to_end(&mut copied).await.unwrap();
+        copy_task.await.unwrap();
+
+        assert_eq!(copied, expected);
+    }
+
     #[tokio::test]
     async fn test_remote_disk_disk_id() {
         let url = url::Url::parse("http://remote-server:9000").unwrap();
@@ -1579,6 +1654,30 @@ mod tests {
         assert!(cleared_id.is_none());
     }
 
+    #[tokio::test]
+    async fn test_remote_disk_ref_prefers_disk_id() {
+        let url = url::Url::parse("http://remote-server:9000").unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+        let disk_option = DiskOption {
+            cleanup: false,
+            health_check: false,
+        };
+
+        let remote_disk = RemoteDisk::new(&endpoint, &disk_option).await.unwrap();
+        assert_eq!(remote_disk.disk_ref().await, endpoint.to_string());
+
+        let disk_id = Uuid::new_v4();
+        remote_disk.set_disk_id(Some(disk_id)).await.unwrap();
+
+        assert_eq!(remote_disk.disk_ref().await, disk_id.to_string());
+    }
+
     #[tokio::test]
     async fn test_remote_disk_endpoints_with_different_schemes() {
         let test_cases = vec![

crates/ecstore/src/rpc/remote_locker.rs

@@ -52,6 +52,7 @@ impl RemoteClient {
             metadata: LockMetadata::default(),
             priority: LockPriority::Normal,
             deadlock_detection: false,
+            suppress_contention_logs: false,
         }
     }
 

crates/ecstore/src/set_disk.rs

@@ -121,6 +121,21 @@ use uuid::Uuid;
 
 pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
 pub const MAX_PARTS_COUNT: usize = 10000;
+
+/// Get the duplex buffer size from environment variable or use default.
+///
+/// This function reads `RUSTFS_DUPLEX_BUFFER_SIZE` environment variable
+/// to allow runtime configuration of the duplex pipe buffer size.
+/// A larger buffer (e.g., 4MB) helps prevent backpressure-related hangs
+/// when reading large objects (20-26MB) under high concurrency.
+///
+/// Default: 4MB (4 * 1024 * 1024 bytes)
+pub fn get_duplex_buffer_size() -> usize {
+    rustfs_utils::get_env_usize(
+        rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE,
+        rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE,
+    )
+}
 const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1);
 const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
 
@@ -136,7 +151,72 @@ mod write;
 /// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
 /// Defaults to 30 seconds if not set or invalid
 pub fn get_lock_acquire_timeout() -> Duration {
-    Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT,
+        rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT,
+    ))
+}
+
+/// Check if lock optimization is enabled.
+/// When enabled, read locks are released after metadata read instead of
+/// being held for the entire data transfer duration.
+pub fn is_lock_optimization_enabled() -> bool {
+    rustfs_utils::get_env_bool(
+        rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE,
+        rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE,
+    )
+}
+
+/// Check if deadlock detection is enabled.
+/// When enabled, lock operations are recorded for deadlock analysis.
+pub fn is_deadlock_detection_enabled() -> bool {
+    rustfs_utils::get_env_bool(
+        rustfs_config::ENV_OBJECT_DEADLOCK_DETECTION_ENABLE,
+        rustfs_config::DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE,
+    )
+}
+
+/// Record a lock acquisition for deadlock detection.
+/// This records detailed lock information for deadlock analysis.
+/// Returns the lock_id for later release tracking.
+#[inline]
+fn record_lock_acquire(bucket: &str, object: &str, lock_type: &str) -> String {
+    let lock_id = format!("{}:{}", bucket, object);
+
+    if !is_deadlock_detection_enabled() {
+        return lock_id;
+    }
+
+    let request_id = format!("get-{}-{}", bucket, object);
+    let resource = format!("{}/{}", bucket, object);
+
+    // Log with structured fields for analysis
+    debug!(
+        request_id = %request_id,
+        lock_id = %lock_id,
+        lock_type = %lock_type,
+        resource = %resource,
+        "Lock acquired for deadlock tracking"
+    );
+
+    lock_id
+}
+
+/// Record a lock release for deadlock detection.
+#[inline]
+fn record_lock_release(bucket: &str, object: &str, lock_id: &str, lock_type: &str) {
+    if !is_deadlock_detection_enabled() {
+        return;
+    }
+
+    let request_id = format!("get-{}-{}", bucket, object);
+
+    debug!(
+        request_id = %request_id,
+        lock_id = %lock_id,
+        lock_type = %lock_type,
+        "Lock released for deadlock tracking"
+    );
 }
 
 fn build_tiered_decommission_file_info(
@@ -451,20 +531,44 @@ impl ObjectIO for SetDisks {
         h: HeaderMap,
         opts: &ObjectOptions,
     ) -> Result<GetObjectReader> {
+        // Check if lock optimization is enabled
+        // When enabled, read locks are released after metadata read
+        let lock_optimization_enabled = is_lock_optimization_enabled();
+
         // Acquire a shared read-lock early to protect read consistency
         let read_lock_guard = if !opts.no_lock {
-            Some(
-                self.new_ns_lock(bucket, object)
-                    .await?
-                    .get_read_lock(get_lock_acquire_timeout())
-                    .await
-                    .map_err(|e| {
-                        Error::other(format!(
-                            "Failed to acquire read lock: {}",
-                            self.format_lock_error_from_error(bucket, object, "read", &e)
-                        ))
-                    })?,
-            )
+            let acquire_start = Instant::now();
+
+            // Record lock wait for deadlock detection
+            if is_deadlock_detection_enabled() {
+                debug!(
+                    lock_id = format!("{}:{}", bucket, object),
+                    lock_type = "read",
+                    resource = format!("{}/{}", bucket, object),
+                    "Waiting for read lock"
+                );
+            }
+
+            let guard = self
+                .new_ns_lock(bucket, object)
+                .await?
+                .get_read_lock(get_lock_acquire_timeout())
+                .await
+                .map_err(|e| {
+                    Error::other(format!(
+                        "Failed to acquire read lock: {}",
+                        self.format_lock_error_from_error(bucket, object, "read", &e)
+                    ))
+                })?;
+
+            // Record lock acquisition for deadlock detection
+            let _lock_id = record_lock_acquire(bucket, object, "read");
+
+            // Record lock statistics
+            metrics::counter!("rustfs.lock.acquire.total", "type" => "read").increment(1);
+            metrics::histogram!("rustfs.lock.acquire.duration.seconds").record(acquire_start.elapsed().as_secs_f64());
+
+            Some(guard)
         } else {
             None
         };
@@ -512,7 +616,28 @@ impl ObjectIO for SetDisks {
             return Ok(gr);
         }
 
-        let (rd, wd) = tokio::io::duplex(DEFAULT_READ_BUFFER_SIZE);
+        // Lock optimization: release read lock after metadata read if enabled
+        // This reduces lock contention by not holding the lock during data transfer
+        let read_lock_guard = if lock_optimization_enabled {
+            // Record lock release for deadlock detection
+            if read_lock_guard.is_some() {
+                let lock_id = format!("{}:{}", bucket, object);
+                record_lock_release(bucket, object, &lock_id, "read");
+
+                // Record early lock release statistics
+                metrics::counter!("rustfs.lock.release.early.total", "type" => "read").increment(1);
+            }
+            // Explicitly drop the lock guard to release the lock early
+            drop(read_lock_guard);
+            debug!(bucket, object, "Lock optimization: released read lock after metadata read");
+            None
+        } else {
+            read_lock_guard
+        };
+
+        let duplex_buffer_size = get_duplex_buffer_size();
+        let (rd, wd) = tokio::io::duplex(duplex_buffer_size);
+        debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer");
 
         let (reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h)?;
 
@@ -523,9 +648,10 @@ impl ObjectIO for SetDisks {
         let pool_index = self.pool_index;
         let skip_verify = opts.skip_verify_bitrot;
         // Move the read-lock guard into the task so it lives for the duration of the read
+        // Note: when lock optimization is enabled, read_lock_guard is None
         // let _guard_to_hold = _read_lock_guard; // moved into closure below
         tokio::spawn(async move {
-            let _guard = read_lock_guard; // keep guard alive until task ends
+            let _guard = read_lock_guard; // keep guard alive until task ends (None if optimization enabled)
             let mut writer = wd;
             if let Err(e) = Self::get_object_with_fileinfo(
                 &bucket,
@@ -751,7 +877,7 @@ impl ObjectIO for SetDisks {
             pfi.metadata = user_defined.clone();
             if is_inline_buffer {
                 if let Some(writer) = writers[i].take() {
-                    pfi.data = Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default());
+                    pfi.data = Some(writer.into_inline_data().map(Bytes::from).unwrap_or_default());
                 }
 
                 pfi.set_inline_data();
@@ -1093,7 +1219,7 @@ impl ObjectOperations for SetDisks {
         let mut unique_objects: HashSet<String> = HashSet::new();
         for dobj in &objects {
             if unique_objects.insert(dobj.object_name.clone()) {
-                batch = batch.add_write_lock(rustfs_lock::ObjectKey::new(bucket, dobj.object_name.clone()));
+                batch = batch.add_write_lock(ObjectKey::new(bucket, dobj.object_name.clone()));
             }
         }
 
@@ -1817,19 +1943,54 @@ impl ObjectOperations for SetDisks {
         //}
 
         let mut uploaded_parts: Vec<CompletePart> = vec![];
-        let rs: Option<HTTPRangeSpec> = None;
-        let gr = get_transitioned_object_reader(bucket, object, &rs, &HeaderMap::new(), &oi, opts).await;
-        if let Err(err) = gr {
-            return set_restore_header_fn(&mut oi, Some(StorageError::Io(err))).await;
-        }
-        let gr = gr.unwrap();
-
-        for part_info in &oi.parts {
-            let reader = BufReader::new(Cursor::new(vec![] /*gr.stream*/));
+        let parts = oi.parts.clone();
+        let mut part_offset: i64 = 0;
+        for part_info in &parts {
+            let mut part_opts = opts.clone();
+            part_opts.part_number = Some(part_info.number);
+            if part_info.actual_size <= 0 {
+                return set_restore_header_fn(
+                    &mut oi,
+                    Some(Error::other(format!("invalid multipart restore part size {}", part_info.actual_size))),
+                )
+                .await;
+            }
+            let part_end = match part_offset.checked_add(part_info.actual_size - 1) {
+                Some(end) => end,
+                None => {
+                    return set_restore_header_fn(
+                        &mut oi,
+                        Some(Error::other("multipart restore part range overflow".to_string())),
+                    )
+                    .await;
+                }
+            };
+            let rs = Some(HTTPRangeSpec {
+                is_suffix_length: false,
+                start: part_offset,
+                end: part_end,
+            });
+            part_offset = match part_end.checked_add(1) {
+                Some(next) => next,
+                None => {
+                    return set_restore_header_fn(
+                        &mut oi,
+                        Some(Error::other("multipart restore part offset overflow".to_string())),
+                    )
+                    .await;
+                }
+            };
+            let gr = match get_transitioned_object_reader(bucket, object, &rs, &HeaderMap::new(), &oi, &part_opts).await {
+                Ok(reader) => reader,
+                Err(err) => {
+                    return set_restore_header_fn(&mut oi, Some(StorageError::Io(err))).await;
+                }
+            };
+            let reader = BufReader::new(gr.stream);
             let hash_reader = HashReader::new(
                 Box::new(WarpReader::new(reader)),
-                part_info.size as i64,
-                part_info.size as i64,
+                part_info.actual_size,
+                part_info.actual_size,
                 None,
                 None,
                 false,
@@ -1842,7 +2003,7 @@ impl ObjectOperations for SetDisks {
             //if let Err(err) = p_info {
             //    return set_restore_header_fn(&mut oi, err).await;
             //}
-            if p_info.size != part_info.size {
+            if p_info.size as i64 != part_info.actual_size {
                 return set_restore_header_fn(
                     &mut oi,
                     Some(Error::other(ObjectApiError::InvalidObjectState(GenericError {
@@ -2757,8 +2918,7 @@ impl MultipartOperations for SetDisks {
         // Build a lookup map for O(1) part resolution instead of O(n) find() in the loop
         // This optimizes from O(n^2) to O(n) when processing many parts
         use std::collections::HashMap;
-        let part_lookup: HashMap<usize, &rustfs_filemeta::ObjectPartInfo> =
-            curr_fi.parts.iter().map(|part| (part.number, part)).collect();
+        let part_lookup: HashMap<usize, &ObjectPartInfo> = curr_fi.parts.iter().map(|part| (part.number, part)).collect();
 
         for (i, p) in uploaded_parts.iter().enumerate() {
             let Some(ext_part) = part_lookup.get(&p.part_num) else {
@@ -3445,12 +3605,11 @@ async fn disks_with_all_parts(
         if (meta.data.is_some() || meta.size == 0) && !meta.parts.is_empty() {
             if let Some(data) = &meta.data {
                 let checksum_info = meta.erasure.get_checksum_info(meta.parts[0].number);
-                let checksum_algo =
-                    if meta.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
-                        rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
-                    } else {
-                        checksum_info.algorithm
-                    };
+                let checksum_algo = if meta.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
+                    HashAlgorithm::HighwayHash256SLegacy
+                } else {
+                    checksum_info.algorithm
+                };
                 let data_len = data.len();
                 let verify_err = bitrot_verify(
                     Box::new(Cursor::new(data.clone())),

crates/ecstore/src/store.rs

@@ -27,7 +27,7 @@ use crate::bucket::utils::check_new_multipart_args;
 use crate::bucket::utils::check_object_args;
 use crate::bucket::utils::check_put_object_args;
 use crate::bucket::utils::check_put_object_part_args;
-use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
+use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict};
 use crate::config::GLOBAL_STORAGE_CLASS;
 use crate::config::storageclass;
 use crate::disk::endpoint::{Endpoint, EndpointType};
@@ -142,8 +142,8 @@ mod rebalance;
 
 use peer::init_local_peer;
 pub use peer::{
-    all_local_disk, all_local_disk_path, find_local_disk, get_disk_infos, get_disk_via_endpoint, has_space_for, init_local_disks,
-    init_lock_clients,
+    all_local_disk, all_local_disk_path, find_local_disk, find_local_disk_by_ref, get_disk_infos, get_disk_via_endpoint,
+    has_space_for, init_local_disks, init_lock_clients,
 };
 
 #[derive(Debug)]

crates/ecstore/src/store/bucket.rs

@@ -13,6 +13,11 @@
 // limitations under the License.
 
 use super::*;
+use crate::bucket::utils::is_meta_bucketname;
+
+fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
+    created != OffsetDateTime::UNIX_EPOCH
+}
 
 impl ECStore {
     #[instrument(skip(self))]
@@ -68,7 +73,9 @@ impl ECStore {
         let mut info = self.peer_sys.get_bucket_info(bucket, opts).await?;
 
         if let Ok(sys) = metadata_sys::get(bucket).await {
-            info.created = Some(sys.created);
+            if should_override_created_from_metadata(sys.created) {
+                info.created = Some(sys.created);
+            }
             info.versioning = sys.versioning();
             info.object_locking = sys.object_locking();
         }
@@ -84,7 +91,9 @@ impl ECStore {
 
         if !opts.no_metadata {
             for bucket in buckets.iter_mut() {
-                if let Ok(created) = metadata_sys::created_at(&bucket.name).await {
+                if let Ok(created) = metadata_sys::created_at(&bucket.name).await
+                    && should_override_created_from_metadata(created)
+                {
                     bucket.created = Some(created);
                 }
             }
@@ -148,3 +157,20 @@ impl ECStore {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::should_override_created_from_metadata;
+    use time::OffsetDateTime;
+
+    #[test]
+    fn should_not_override_when_metadata_created_is_unix_epoch() {
+        assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH));
+    }
+
+    #[test]
+    fn should_override_when_metadata_created_is_valid_time() {
+        let created = OffsetDateTime::from_unix_timestamp(1704067200).expect("valid timestamp");
+        assert!(should_override_created_from_metadata(created));
+    }
+}

crates/ecstore/src/store/init.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use crate::global::is_first_cluster_node_local;
 
 impl ECStore {
     #[allow(clippy::new_ret_no_self)]
@@ -203,6 +204,7 @@ impl ECStore {
         let mut meta = PoolMeta::default();
         meta.load(self.pools[0].clone(), self.pools.clone()).await?;
         let update = meta.validate(self.pools.clone())?;
+        let should_persist_pool_meta = is_first_cluster_node_local().await;
 
         if !update {
             {
@@ -211,7 +213,9 @@ impl ECStore {
             }
         } else {
             let new_meta = PoolMeta::new(&self.pools, &meta);
-            new_meta.save(self.pools.clone()).await?;
+            if should_persist_pool_meta {
+                new_meta.save(self.pools.clone()).await?;
+            }
             {
                 let mut pool_meta = self.pool_meta.write().await;
                 *pool_meta = new_meta;

crates/ecstore/src/store/peer.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 
 pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
     let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await;
@@ -24,6 +25,19 @@ pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
     }
 }
 
+pub async fn find_local_disk_by_ref(disk_ref: &str) -> Option<DiskStore> {
+    if let Some(disk) = find_local_disk(&disk_ref.to_string()).await {
+        return Some(disk);
+    }
+
+    let Ok(disk_id) = Uuid::parse_str(disk_ref) else {
+        return None;
+    };
+
+    let disk_path = GLOBAL_LOCAL_DISK_ID_MAP.read().await.get(&disk_id).cloned()?;
+    find_local_disk(&disk_path).await
+}
+
 pub async fn get_disk_via_endpoint(endpoint: &Endpoint) -> Option<DiskStore> {
     let global_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.read().await;
     if global_set_drives.is_empty() {

crates/ecstore/src/store/rebalance.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use crate::bucket::utils::is_meta_bucketname;
 
 impl ECStore {
     #[instrument(level = "debug", skip(self))]

crates/ecstore/src/tier/tier.rs

@@ -49,6 +49,7 @@ use crate::{
     StorageAPI,
     config::com::{CONFIG_PREFIX, read_config},
     disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
+    global::{get_global_endpoints, is_first_cluster_node_local},
     store::ECStore,
     store_api::{ObjectIO as _, ObjectOptions, PutObjReader},
 };
@@ -1118,7 +1119,15 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
                 }
                 Err(legacy_err) if is_err_config_not_found(&legacy_err) => {
                     warn!("config not found, start to init");
-                    new_and_save_tiering_config(api).await.map_err(io::Error::other)
+                    if is_first_cluster_node_local().await {
+                        new_and_save_tiering_config(api).await.map_err(io::Error::other)
+                    } else {
+                        Ok(TierConfigMgr {
+                            driver_cache: HashMap::new(),
+                            tiers: HashMap::new(),
+                            last_refreshed_at: OffsetDateTime::now_utc(),
+                        })
+                    }
                 }
                 Err(legacy_err) => Err(io::Error::other(legacy_err)),
             }
@@ -1166,7 +1175,14 @@ async fn write_tier_config_to_rustfs<S: StorageAPI>(api: Arc<S>, path: &str, dat
 pub async fn try_migrate_tiering_config<S: StorageAPI>(api: Arc<S>) {
     let target_path = tier_config_path(TIER_CONFIG_FILE);
     if api
-        .get_object_info(RUSTFS_META_BUCKET, &target_path, &ObjectOptions::default())
+        .get_object_info(
+            RUSTFS_META_BUCKET,
+            &target_path,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
         .await
         .is_ok()
     {

crates/filemeta/Cargo.toml

@@ -51,3 +51,6 @@ harness = false
 
 [lints]
 workspace = true
+
+[lib]
+doctest = false

crates/filemeta/src/filemeta.rs

@@ -74,6 +74,14 @@ pub(crate) fn data_key_for_version(version_id: Option<Uuid>) -> String {
     }
 }
 
+fn legacy_data_key_for_version(version_id: Option<Uuid>) -> Option<String> {
+    if version_id.is_none() || version_id == Some(Uuid::nil()) {
+        Some(Uuid::nil().to_string())
+    } else {
+        None
+    }
+}
+
 pub const TRANSITION_COMPLETE: &str = "complete";
 pub const TRANSITION_PENDING: &str = "pending";
 
@@ -164,6 +172,21 @@ impl FileMeta {
         });
     }
 
+    fn find_inline_data_for_version(&self, version_id: Option<Uuid>) -> Result<Option<Vec<u8>>> {
+        let key = data_key_for_version(version_id);
+        if let Some(data) = self.data.find(key.as_str())? {
+            return Ok(Some(data));
+        }
+
+        if let Some(legacy_key) = legacy_data_key_for_version(version_id)
+            && legacy_key != key
+        {
+            return self.data.find(legacy_key.as_str());
+        }
+
+        Ok(None)
+    }
+
     // Find version
     pub fn find_version(&self, vid: Option<Uuid>) -> Result<(usize, FileMetaVersion)> {
         let vid = vid.unwrap_or_default();
@@ -709,10 +732,7 @@ impl FileMeta {
             }
 
             if read_data && fi.inline_data() {
-                fi.data = self
-                    .data
-                    .find(data_key_for_version(fi.version_id).as_str())?
-                    .map(bytes::Bytes::from);
+                fi.data = self.find_inline_data_for_version(fi.version_id)?.map(bytes::Bytes::from);
             }
 
             found_fi = Some(fi);
@@ -1071,6 +1091,28 @@ mod test {
         assert!(!inline_data.is_empty(), "Inline data should not be empty");
     }
 
+    #[test]
+    fn test_into_fileinfo_reads_legacy_nil_uuid_inline_key() {
+        let mut fm = FileMeta::new();
+        let mut fi = FileInfo::new("test", 2, 1);
+        fi.mod_time = Some(OffsetDateTime::now_utc());
+        fi.data = Some(Bytes::from_static(b"legacy-inline"));
+        fi.set_inline_data();
+        fm.add_version(fi).unwrap();
+
+        let current_key = data_key_for_version(Some(Uuid::nil()));
+        let legacy_key = legacy_data_key_for_version(Some(Uuid::nil())).unwrap();
+        let payload = fm.data.find(current_key.as_str()).unwrap().unwrap();
+
+        fm.data.remove_key(current_key.as_str()).unwrap();
+        fm.data.replace(legacy_key.as_str(), payload.clone()).unwrap();
+
+        let restored = fm.into_fileinfo("bucket", "test", "", true, false, true).unwrap();
+
+        assert!(restored.inline_data());
+        assert_eq!(restored.data, Some(Bytes::from(payload)));
+    }
+
     #[test]
     fn test_error_handling_and_recovery() {
         // Test error handling and recovery

crates/filemeta/src/filemeta/inline_data.rs

@@ -43,11 +43,7 @@ impl FileMeta {
         let vid = version_id.unwrap_or_default();
 
         if self.data.entries().unwrap_or_default() > 0
-            && self
-                .data
-                .find(super::data_key_for_version(version_id).as_str())
-                .unwrap_or_default()
-                .is_some()
+            && self.find_inline_data_for_version(version_id).unwrap_or_default().is_some()
         {
             return 0;
         }

crates/heal/Cargo.toml

@@ -50,3 +50,6 @@ tracing-subscriber = { workspace = true }
 tempfile = { workspace = true }
 walkdir = { workspace = true }
 http = { workspace = true }
+
+[lib]
+doctest = false

crates/iam/Cargo.toml

@@ -58,3 +58,6 @@ url = { workspace = true }
 
 [dev-dependencies]
 pollster.workspace = true
+
+[lib]
+doctest = false

crates/iam/src/manager.rs

@@ -25,6 +25,7 @@ use crate::{
 };
 use futures::future::join_all;
 use rustfs_credentials::{Credentials, EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, get_global_action_cred};
+use rustfs_ecstore::global::is_first_cluster_node_local;
 use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq, GroupDesc};
 use rustfs_policy::{
     arn::ARN,
@@ -303,6 +304,10 @@ where
             return Ok(());
         }
 
+        if !is_first_cluster_node_local().await {
+            return Ok(());
+        }
+
         self.api.save_iam_config(IAMFormat::new_version_1(), path).await
     }
     pub async fn get_user(&self, access_key: &str) -> Option<UserIdentity> {

crates/iam/src/oidc.rs

@@ -25,6 +25,8 @@ use openidconnect::{
     PkceCodeVerifier, RedirectUrl, Scope,
 };
 use rustfs_config::oidc::*;
+use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
+use rustfs_ecstore::config::{Config as ServerConfig, KVS, get_global_server_config};
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
 use std::collections::HashMap;
@@ -101,7 +103,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
 // ---- Public types (unchanged API) ----
 
 /// Parsed configuration for a single OIDC provider.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct OidcProviderConfig {
     pub id: String,
     pub enabled: bool,
@@ -120,6 +122,26 @@ pub struct OidcProviderConfig {
     pub username_claim: String,
 }
 
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum OidcProviderConfigSource {
+    Env,
+    Persisted,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SourcedOidcProviderConfig {
+    pub config: OidcProviderConfig,
+    pub source: OidcProviderConfigSource,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct OidcProviderValidationResult {
+    pub issuer: String,
+    pub authorization_endpoint: String,
+    pub token_endpoint: Option<String>,
+}
+
 /// Summary info about a provider, returned to the console.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct OidcProviderSummary {
@@ -170,11 +192,12 @@ impl OidcSys {
     /// Parse environment variables and discover all configured OIDC providers.
     pub async fn new() -> Result<Self, String> {
         let http_client = ReqwestHttpClient(reqwest::Client::new());
-        let parsed_configs = Self::parse_env_configs();
+        let parsed_configs = load_effective_oidc_provider_configs(get_global_server_config().as_ref());
         let mut configs = HashMap::new();
         let mut provider_states = HashMap::new();
 
-        for config in parsed_configs {
+        for sourced_config in parsed_configs {
+            let config = sourced_config.config;
             if !config.enabled {
                 info!("OIDC provider '{}' is disabled, skipping", config.id);
                 continue;
@@ -620,6 +643,33 @@ impl OidcSys {
         configs
     }
 
+    fn parse_persisted_configs(cfg: &ServerConfig) -> Vec<OidcProviderConfig> {
+        let Some(subsystem) = cfg.0.get(IDENTITY_OPENID_SUB_SYS) else {
+            return Vec::new();
+        };
+
+        let mut configs = Vec::new();
+        let mut provider_ids: Vec<String> = subsystem.keys().cloned().collect();
+        provider_ids.sort();
+
+        for raw_id in provider_ids {
+            let Some(kvs) = subsystem.get(&raw_id) else {
+                continue;
+            };
+
+            let id = if raw_id == DEFAULT_DELIMITER {
+                "default"
+            } else {
+                raw_id.as_str()
+            };
+            if let Some(config) = Self::parse_single_persisted_provider(kvs, id) {
+                configs.push(config);
+            }
+        }
+
+        configs
+    }
+
     /// Parse a single provider's config from env vars with the given suffix.
     fn parse_single_provider(env_suffix: &str, id: &str) -> Option<OidcProviderConfig> {
         let get_env = |base: &str| -> String { std::env::var(format!("{base}{env_suffix}")).unwrap_or_default() };
@@ -716,6 +766,68 @@ impl OidcSys {
         })
     }
 
+    fn parse_single_persisted_provider(kvs: &KVS, id: &str) -> Option<OidcProviderConfig> {
+        let config_url = kvs.get(OIDC_CONFIG_URL);
+        if config_url.is_empty() {
+            return None;
+        }
+
+        let enabled = kvs
+            .lookup(ENABLE_KEY)
+            .unwrap_or_else(|| EnableState::Off.to_string())
+            .parse::<EnableState>()
+            .map(|s| s.is_enabled())
+            .unwrap_or(false);
+
+        let scopes_str = kvs.get(OIDC_SCOPES);
+        let scopes = if scopes_str.is_empty() {
+            OIDC_DEFAULT_SCOPES.split(',').map(String::from).collect()
+        } else {
+            scopes_str.split(',').map(|s| s.trim().to_string()).collect()
+        };
+
+        let redirect_uri_dynamic = kvs
+            .lookup(OIDC_REDIRECT_URI_DYNAMIC)
+            .unwrap_or_else(|| EnableState::On.to_string())
+            .parse::<EnableState>()
+            .map(|s| s.is_enabled())
+            .unwrap_or(true);
+
+        let claim_name = kvs
+            .lookup(OIDC_CLAIM_NAME)
+            .unwrap_or_else(|| OIDC_DEFAULT_CLAIM_NAME.to_string());
+        let groups_claim = kvs
+            .lookup(OIDC_GROUPS_CLAIM)
+            .unwrap_or_else(|| OIDC_DEFAULT_GROUPS_CLAIM.to_string());
+        let email_claim = kvs
+            .lookup(OIDC_EMAIL_CLAIM)
+            .unwrap_or_else(|| OIDC_DEFAULT_EMAIL_CLAIM.to_string());
+        let username_claim = kvs
+            .lookup(OIDC_USERNAME_CLAIM)
+            .unwrap_or_else(|| OIDC_DEFAULT_USERNAME_CLAIM.to_string());
+        let display_name = kvs.lookup(OIDC_DISPLAY_NAME).unwrap_or_else(|| id.to_string());
+        let redirect_uri = kvs.lookup(OIDC_REDIRECT_URI).filter(|v| !v.is_empty());
+        let client_secret = kvs.lookup(OIDC_CLIENT_SECRET).filter(|v| !v.is_empty());
+
+        Some(OidcProviderConfig {
+            id: id.to_string(),
+            enabled,
+            config_url,
+            client_id: kvs.get(OIDC_CLIENT_ID),
+            client_secret,
+            scopes,
+            redirect_uri,
+            redirect_uri_dynamic,
+            claim_name,
+            claim_prefix: kvs.get(OIDC_CLAIM_PREFIX),
+            role_policy: kvs.get(OIDC_ROLE_POLICY),
+            display_name,
+            groups_claim,
+            email_claim,
+            username_claim,
+        })
+    }
+
     /// Perform OIDC discovery for a provider.
     /// `discover_async` fetches the discovery document and JWKS in one step.
     async fn discover_provider(config: &OidcProviderConfig, http_client: &ReqwestHttpClient) -> Result<ProviderState, String> {
@@ -736,6 +848,64 @@ impl OidcSys {
     }
 }
 
+pub fn load_oidc_provider_configs_from_env() -> Vec<OidcProviderConfig> {
+    OidcSys::parse_env_configs()
+}
+
+pub fn load_oidc_provider_configs_from_server_config(cfg: &ServerConfig) -> Vec<OidcProviderConfig> {
+    OidcSys::parse_persisted_configs(cfg)
+}
+
+pub fn merge_oidc_provider_configs(
+    env_configs: Vec<OidcProviderConfig>,
+    persisted_configs: Vec<OidcProviderConfig>,
+) -> Vec<SourcedOidcProviderConfig> {
+    let mut effective = HashMap::new();
+
+    for config in persisted_configs {
+        effective.insert(
+            config.id.clone(),
+            SourcedOidcProviderConfig {
+                config,
+                source: OidcProviderConfigSource::Persisted,
+            },
+        );
+    }
+
+    for config in env_configs {
+        effective.insert(
+            config.id.clone(),
+            SourcedOidcProviderConfig {
+                config,
+                source: OidcProviderConfigSource::Env,
+            },
+        );
+    }
+
+    let mut configs: Vec<SourcedOidcProviderConfig> = effective.into_values().collect();
+    configs.sort_by(|lhs, rhs| lhs.config.id.cmp(&rhs.config.id));
+    configs
+}
+
+pub fn load_effective_oidc_provider_configs(server_config: Option<&ServerConfig>) -> Vec<SourcedOidcProviderConfig> {
+    let env_configs = load_oidc_provider_configs_from_env();
+    let persisted_configs = server_config
+        .map(load_oidc_provider_configs_from_server_config)
+        .unwrap_or_default();
+    merge_oidc_provider_configs(env_configs, persisted_configs)
+}
+
+pub async fn validate_oidc_provider_config(config: &OidcProviderConfig) -> Result<OidcProviderValidationResult, String> {
+    let http_client = ReqwestHttpClient(reqwest::Client::new());
+    let state = OidcSys::discover_provider(config, &http_client).await?;
+
+    Ok(OidcProviderValidationResult {
+        issuer: state.metadata.issuer().to_string(),
+        authorization_endpoint: state.metadata.authorization_endpoint().to_string(),
+        token_endpoint: state.metadata.token_endpoint().map(ToString::to_string),
+    })
+}
+
 // --- Helper functions ---
 
 fn normalize_issuer(raw: &str) -> Option<(String, String, u16, String)> {
@@ -761,14 +931,21 @@ fn normalize_config_url(config_url: &str) -> Result<String, String> {
         return Err(format!("invalid config_url scheme: {}", url.scheme()));
     }
     let host = url.host_str().ok_or_else(|| "config_url missing host".to_string())?;
-    let path = url.path().trim_end_matches('/');
+    let path = url.path();
 
-    if path.contains("/.well-known/") && !path.ends_with("/.well-known/openid-configuration") {
+    // Strip `/.well-known/openid-configuration` (with optional trailing slash) if present.
+    // Everything else is preserved exactly so the issuer URL matches the provider's discovery
+    // document (e.g. Authentik includes a trailing slash, Keycloak does not).
+    let normalized_path = path
+        .strip_suffix('/')
+        .unwrap_or(path)
+        .strip_suffix("/.well-known/openid-configuration")
+        .unwrap_or(if path == "/" { "" } else { path });
+
+    if normalized_path.contains("/.well-known/") {
         return Err("config_url uses an unsupported .well-known discovery URL".into());
     }
 
-    let normalized_path = path.strip_suffix("/.well-known/openid-configuration").unwrap_or(path);
-
     let mut issuer = format!("{}://{host}", url.scheme());
     if let Some(port) = url.port() {
         issuer.push(':');
@@ -886,20 +1063,57 @@ mod tests {
 
     #[test]
     fn test_normalize_config_url() {
+        // --- Well-known suffix stripping ---
+        // Bare well-known URL → stripped to just the host
         assert_eq!(
             normalize_config_url("https://idp.example.com/.well-known/openid-configuration").unwrap(),
             "https://idp.example.com"
         );
+        // Trailing slash after well-known suffix is also stripped
         assert_eq!(
             normalize_config_url("https://idp.example.com/.well-known/openid-configuration/").unwrap(),
             "https://idp.example.com"
         );
+        // Well-known under a sub-path (Keycloak realms)
+        assert_eq!(
+            normalize_config_url("https://keycloak.example.com/realms/myrealm/.well-known/openid-configuration").unwrap(),
+            "https://keycloak.example.com/realms/myrealm"
+        );
+
+        // --- Providers WITHOUT trailing slash (Keycloak, Auth0, Okta, Google) ---
+        assert_eq!(
+            normalize_config_url("https://keycloak.example.com/realms/myrealm").unwrap(),
+            "https://keycloak.example.com/realms/myrealm"
+        );
         assert_eq!(
             normalize_config_url("https://idp.example.com/custom/realm").unwrap(),
             "https://idp.example.com/custom/realm"
         );
+
+        // --- Providers WITH trailing slash (Authentik) ---
+        assert_eq!(
+            normalize_config_url("https://auth.example.com/application/o/myapp/").unwrap(),
+            "https://auth.example.com/application/o/myapp/"
+        );
+
+        // --- Root-level issuer (bare host) ---
+        assert_eq!(normalize_config_url("https://idp.example.com").unwrap(), "https://idp.example.com");
+        assert_eq!(normalize_config_url("https://idp.example.com/").unwrap(), "https://idp.example.com");
+
+        // --- Custom port ---
+        assert_eq!(
+            normalize_config_url("https://idp.example.com:8443/auth/realms/test").unwrap(),
+            "https://idp.example.com:8443/auth/realms/test"
+        );
+        assert_eq!(
+            normalize_config_url("http://localhost:8080/application/o/app/").unwrap(),
+            "http://localhost:8080/application/o/app/"
+        );
+
+        // --- Error cases ---
         assert!(normalize_config_url("https://idp.example.com/.well-known/invalid").is_err());
         assert!(normalize_config_url("gopher://idp.example.com").is_err());
+        assert!(normalize_config_url("not-a-url").is_err());
     }
 
     #[test]
@@ -975,6 +1189,59 @@ mod tests {
         assert!(config.is_none());
     }
 
+    #[test]
+    fn test_parse_persisted_provider_config() {
+        let mut cfg = ServerConfig::new();
+        let mut kvs = KVS(vec![
+            rustfs_ecstore::config::KV {
+                key: ENABLE_KEY.to_string(),
+                value: EnableState::Off.to_string(),
+                hidden_if_empty: false,
+            },
+            rustfs_ecstore::config::KV {
+                key: OIDC_CONFIG_URL.to_string(),
+                value: String::new(),
+                hidden_if_empty: false,
+            },
+            rustfs_ecstore::config::KV {
+                key: OIDC_CLIENT_ID.to_string(),
+                value: String::new(),
+                hidden_if_empty: false,
+            },
+        ]);
+        kvs.insert(
+            OIDC_CONFIG_URL.to_string(),
+            "https://example.com/.well-known/openid-configuration".to_string(),
+        );
+        kvs.insert(OIDC_CLIENT_ID.to_string(), "console".to_string());
+        kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+
+        cfg.0
+            .entry(IDENTITY_OPENID_SUB_SYS.to_string())
+            .or_default()
+            .insert(DEFAULT_DELIMITER.to_string(), kvs);
+
+        let parsed = OidcSys::parse_persisted_configs(&cfg);
+        assert_eq!(parsed.len(), 1);
+        assert_eq!(parsed[0].id, "default");
+        assert_eq!(parsed[0].client_id, "console");
+        assert!(parsed[0].enabled);
+    }
+
+    #[test]
+    fn test_merge_oidc_provider_configs_prefers_env() {
+        let mut persisted = test_config("default");
+        persisted.display_name = "Persisted".to_string();
+
+        let mut env = test_config("default");
+        env.display_name = "Environment".to_string();
+
+        let merged = merge_oidc_provider_configs(vec![env], vec![persisted]);
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].config.display_name, "Environment");
+        assert_eq!(merged[0].source, OidcProviderConfigSource::Env);
+    }
+
     #[test]
     fn test_oidc_sys_empty() {
         let sys = OidcSys::empty();

crates/iam/src/store/object.rs

@@ -25,7 +25,7 @@ use rustfs_ecstore::store_api::{ListOperations as _, ObjectInfoOrErr, WalkOption
 use rustfs_ecstore::{
     config::{
         RUSTFS_CONFIG_PREFIX,
-        com::{delete_config, read_config, read_config_with_metadata, save_config},
+        com::{delete_config, read_config, read_config_no_lock, read_config_with_metadata, save_config},
     },
     store::ECStore,
     store_api::{ObjectInfo, ObjectOptions},
@@ -391,7 +391,7 @@ impl ObjectStore {
         // If it doesn't exist, the system bucket or metadata is not ready.
         let probe_path = format!("{}/format.json", *IAM_CONFIG_PREFIX);
 
-        match read_config(self.object_api.clone(), &probe_path).await {
+        match read_config_no_lock(self.object_api.clone(), &probe_path).await {
             Ok(_) => Ok(()),
             Err(rustfs_ecstore::error::StorageError::ConfigNotFound) => Err(Error::other(format!(
                 "Storage metadata not ready: probe object '{}' not found (expected IAM config to be initialized)",

crates/iam/src/sys.rs

@@ -794,6 +794,15 @@ impl<T: Store> IamSys<T> {
                     return combined.is_allowed(args).await;
                 }
             }
+
+            if args.deny_only {
+                let combined_policy = Policy::default();
+                let (has_session_policy, is_allowed_sp) = is_allowed_by_session_policy(args);
+                if has_session_policy {
+                    return is_allowed_sp && combined_policy.is_allowed(args).await;
+                }
+                return combined_policy.is_allowed(args).await;
+            }
             return false;
         }
 
@@ -803,9 +812,14 @@ impl<T: Store> IamSys<T> {
             } else {
                 let (a, c) = self.store.merge_policies(&policies.join(",")).await;
                 if a.is_empty() {
-                    return false;
+                    if args.deny_only {
+                        Policy::default()
+                    } else {
+                        return false;
+                    }
+                } else {
+                    c
                 }
-                c
             }
         };
 
@@ -1035,17 +1049,20 @@ mod tests {
     use rustfs_credentials::Credentials;
     use rustfs_policy::auth::UserIdentity;
     use rustfs_policy::policy::Args;
-    use rustfs_policy::policy::action::{Action, S3Action};
+    use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
+    use serde_json::Value;
     use std::collections::HashMap;
     use time::OffsetDateTime;
 
-    /// Mock Store that populates cache in load_all so STS temp credentials
-    /// without args.groups still get group-attached policies via parent user (groups fallback).
+    /// Mock Store for STS tests: either group-attached policies via parent user, or no IAM policies.
     #[derive(Clone)]
-    struct StsGroupsFallbackMockStore;
+    struct StsTestMockStore {
+        /// When true, parent user has no groups and no mapped policies (empty `policy_db_get`).
+        empty_policies: bool,
+    }
 
     #[async_trait::async_trait]
-    impl Store for StsGroupsFallbackMockStore {
+    impl Store for StsTestMockStore {
         fn has_watcher(&self) -> bool {
             false
         }
@@ -1175,14 +1192,48 @@ mod tests {
         }
 
         async fn load_all(&self, cache: &Cache) -> Result<()> {
-            const PARENT_USER: &str = "sts-fallback-test-parent";
-            const GROUP_NAME: &str = "testgroup";
-
             let policy_docs = get_default_policyes();
             cache
                 .policy_docs
                 .store(Arc::new(CacheEntity::new(policy_docs).update_load_time()));
 
+            if self.empty_policies {
+                const PARENT_USER: &str = "sts-empty-parent-policy-test";
+                let creds = Credentials {
+                    access_key: PARENT_USER.to_string(),
+                    secret_key: "longenoughsecret".to_string(),
+                    session_token: String::new(),
+                    expiration: None,
+                    status: "on".to_string(),
+                    parent_user: String::new(),
+                    groups: None,
+                    claims: None,
+                    name: None,
+                    description: None,
+                };
+                let parent_identity = UserIdentity {
+                    version: 1,
+                    credentials: creds,
+                    update_at: Some(OffsetDateTime::now_utc()),
+                };
+                let mut users = HashMap::new();
+                users.insert(PARENT_USER.to_string(), parent_identity);
+                cache.users.store(Arc::new(CacheEntity::new(users).update_load_time()));
+
+                cache.groups.store(Arc::new(CacheEntity::default().update_load_time()));
+                cache
+                    .group_policies
+                    .store(Arc::new(CacheEntity::default().update_load_time()));
+                cache.user_policies.store(Arc::new(CacheEntity::default().update_load_time()));
+                cache.sts_accounts.store(Arc::new(CacheEntity::default().update_load_time()));
+                cache.sts_policies.store(Arc::new(CacheEntity::default().update_load_time()));
+                cache.build_user_group_memberships();
+                return Ok(());
+            }
+
+            const PARENT_USER: &str = "sts-fallback-test-parent";
+            const GROUP_NAME: &str = "testgroup";
+
             let creds = Credentials {
                 access_key: PARENT_USER.to_string(),
                 secret_key: "longenoughsecret".to_string(),
@@ -1231,7 +1282,7 @@ mod tests {
     /// would be denied.
     #[tokio::test]
     async fn test_sts_groups_fallback_temp_creds_receive_parent_group_policies() {
-        let store = StsGroupsFallbackMockStore;
+        let store = StsTestMockStore { empty_policies: false };
         let cache_manager = IamCache::new(store).await;
         let iam_sys = IamSys::new(cache_manager);
 
@@ -1257,13 +1308,93 @@ mod tests {
         );
     }
 
+    /// Regression: `deny_only` with empty IAM policies must still evaluate `sessionPolicy-extracted`
+    /// so session policy Deny cannot be bypassed (see PR #2250 review).
+    #[tokio::test]
+    async fn test_sts_deny_only_session_policy_deny_blocks_when_iam_policies_empty() {
+        let store = StsTestMockStore { empty_policies: true };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let parent_user = "sts-empty-parent-policy-test";
+        let session_policy_json = r#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Deny",
+      "Action": ["admin:CreateUser"],
+      "Resource": ["arn:aws:s3:::*"]
+    }
+  ]
+}"#;
+        let mut claims = HashMap::new();
+        claims.insert(SESSION_POLICY_NAME_EXTRACTED.to_string(), Value::String(session_policy_json.to_string()));
+        let groups: Option<Vec<String>> = None;
+        let args = Args {
+            account: parent_user,
+            groups: &groups,
+            action: Action::AdminAction(AdminAction::CreateUserAdminAction),
+            bucket: "",
+            conditions: &HashMap::new(),
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: true,
+        };
+
+        let allowed = iam_sys.is_allowed_sts(&args, parent_user).await;
+        assert!(
+            !allowed,
+            "session policy Deny must be evaluated even when IAM policies are empty and deny_only is set"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_sts_deny_only_session_policy_allow_when_no_deny_on_action() {
+        let store = StsTestMockStore { empty_policies: true };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let parent_user = "sts-empty-parent-policy-test";
+        let session_policy_json = r#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["s3:GetObject"],
+      "Resource": ["arn:aws:s3:::bucket/*"]
+    }
+  ]
+}"#;
+        let mut claims = HashMap::new();
+        claims.insert(SESSION_POLICY_NAME_EXTRACTED.to_string(), Value::String(session_policy_json.to_string()));
+        let groups: Option<Vec<String>> = None;
+        let args = Args {
+            account: parent_user,
+            groups: &groups,
+            action: Action::AdminAction(AdminAction::CreateUserAdminAction),
+            bucket: "",
+            conditions: &HashMap::new(),
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: true,
+        };
+
+        let allowed = iam_sys.is_allowed_sts(&args, parent_user).await;
+        assert!(
+            allowed,
+            "deny_only with no matching Deny in session policy should still allow self-service-style checks"
+        );
+    }
+
     /// Regression test for cross-node IAM notifications:
     /// `load_user` must populate user cache, and regular-user mapped policy must be written to
     /// `user_policies` (not `sts_policies`), otherwise list-users and bucket-scoped user listing
     /// may miss users on follower nodes.
     #[tokio::test]
     async fn test_load_user_notification_populates_user_and_policy_caches() {
-        let store = StsGroupsFallbackMockStore;
+        let store = StsTestMockStore { empty_policies: false };
         let cache_manager = IamCache::new(store).await;
         let iam_sys = IamSys::new(cache_manager);
 

crates/lock/Cargo.toml

@@ -43,3 +43,6 @@ parking_lot.workspace = true
 smallvec.workspace = true
 smartstring.workspace = true
 crossbeam-queue = { workspace = true }
+
+[lib]
+doctest = false

crates/lock/src/distributed_lock.rs

@@ -230,7 +230,21 @@ impl DistributedLock {
         } else {
             // Check if it's a timeout or quorum failure
             if let Some(error_msg) = &resp.error {
-                warn!("acquire_lock_quorum error: {}", error_msg);
+                if request.suppress_contention_logs {
+                    tracing::debug!(
+                        resource = %request.resource,
+                        owner = %request.owner,
+                        "acquire_lock_quorum contention: {}",
+                        error_msg
+                    );
+                } else {
+                    warn!(
+                        resource = %request.resource,
+                        owner = %request.owner,
+                        "acquire_lock_quorum error: {}",
+                        error_msg
+                    );
+                }
                 if error_msg.contains("quorum") {
                     // This is a quorum failure - return appropriate error
                     // Extract achieved count from error message or use individual_locks.len()
@@ -266,6 +280,21 @@ impl DistributedLock {
         self.acquire_guard(&req).await
     }
 
+    /// Convenience: acquire exclusive lock with expected contention logs suppressed
+    pub async fn lock_guard_quiet(
+        &self,
+        resource: ObjectKey,
+        owner: &str,
+        timeout: Duration,
+        ttl: Duration,
+    ) -> Result<Option<DistributedLockGuard>> {
+        let req = LockRequest::new(resource, LockType::Exclusive, owner)
+            .with_acquire_timeout(timeout)
+            .with_ttl(ttl)
+            .with_suppress_contention_logs(true);
+        self.acquire_guard(&req).await
+    }
+
     /// Convenience: acquire shared lock as a guard
     pub async fn rlock_guard(
         &self,
@@ -307,11 +336,24 @@ impl DistributedLock {
                             individual_locks.push((lock_info.id.clone(), self.clients[idx].clone()));
                         }
                     } else {
-                        tracing::warn!(
-                            "Failed to acquire lock on client from response: {}, error: {}",
-                            idx,
-                            resp.error.unwrap_or_else(|| "unknown error".to_string())
-                        );
+                        let error = resp.error.unwrap_or_else(|| "unknown error".to_string());
+                        if request.suppress_contention_logs {
+                            tracing::debug!(
+                                resource = %request.resource,
+                                owner = %request.owner,
+                                "Failed to acquire lock on client from response: {}, error: {}",
+                                idx,
+                                error
+                            );
+                        } else {
+                            tracing::warn!(
+                                resource = %request.resource,
+                                owner = %request.owner,
+                                "Failed to acquire lock on client from response: {}, error: {}",
+                                idx,
+                                error
+                            );
+                        }
                     }
                 }
                 Err(e) => {

crates/lock/src/namespace/mod.rs

@@ -58,6 +58,16 @@ impl NamespaceLockWrapper {
         self.lock.get_write_lock(self.resource.clone(), &self.owner, timeout).await
     }
 
+    /// Acquire write lock with expected contention logs suppressed
+    pub async fn get_write_lock_quiet(
+        &self,
+        timeout: Duration,
+    ) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
+        self.lock
+            .get_write_lock_quiet(self.resource.clone(), &self.owner, timeout)
+            .await
+    }
+
     /// Acquire read lock (shared lock) with timeout
     /// Returns the guard if acquisition succeeds, or an error if it fails
     pub async fn get_read_lock(&self, timeout: Duration) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
@@ -237,6 +247,29 @@ impl NamespaceLock {
         }
     }
 
+    /// Acquire write lock while suppressing expected contention warnings
+    pub async fn get_write_lock_quiet(
+        &self,
+        resource: ObjectKey,
+        owner: &str,
+        timeout: Duration,
+    ) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
+        let ttl = crate::fast_lock::DEFAULT_LOCK_TIMEOUT;
+        let resource_str = format!("{}", resource);
+        match self {
+            Self::Distributed(lock) => match lock.lock_guard_quiet(resource, owner, timeout, ttl).await {
+                Ok(Some(guard)) => Ok(NamespaceLockGuard::Standard(guard)),
+                Ok(None) => Err(crate::error::LockError::timeout(resource_str, timeout)),
+                Err(e) => Err(e),
+            },
+            Self::Local(lock) => match lock.lock_guard(resource, owner, timeout, ttl).await {
+                Ok(Some(guard)) => Ok(NamespaceLockGuard::Fast(guard)),
+                Ok(None) => Err(crate::error::LockError::timeout(resource_str, timeout)),
+                Err(e) => Err(e),
+            },
+        }
+    }
+
     /// Acquire read lock (shared lock) with timeout
     /// Returns the guard if acquisition succeeds, or an error if it fails
     pub async fn get_read_lock(

crates/lock/src/types.rs

@@ -225,6 +225,8 @@ pub struct LockRequest {
     pub priority: LockPriority,
     /// Deadlock detection
     pub deadlock_detection: bool,
+    /// Suppress warning logs for expected contention failures
+    pub suppress_contention_logs: bool,
 }
 
 impl LockRequest {
@@ -240,6 +242,7 @@ impl LockRequest {
             metadata: LockMetadata::default(),
             priority: LockPriority::default(),
             deadlock_detection: false,
+            suppress_contention_logs: false,
         }
     }
 
@@ -272,6 +275,12 @@ impl LockRequest {
         self.deadlock_detection = enabled;
         self
     }
+
+    /// Suppress warning logs for expected contention failures
+    pub fn with_suppress_contention_logs(mut self, enabled: bool) -> Self {
+        self.suppress_contention_logs = enabled;
+        self
+    }
 }
 
 /// Lock response structure

crates/madmin/Cargo.toml

@@ -35,3 +35,6 @@ hyper.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 time.workspace = true
+
+[lib]
+doctest = false

crates/madmin/src/user.rs

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as DeError};
 use serde_json::Value;
 use serde_json::value::RawValue;
 use std::collections::HashMap;
 use time::OffsetDateTime;
+use time::format_description::well_known::Rfc3339;
 
 use crate::BackendInfo;
 
@@ -492,6 +493,34 @@ impl<'de> Deserialize<'de> for SRSessionPolicy {
     }
 }
 
+fn deserialize_vec_or_default<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    Ok(Option::<Vec<String>>::deserialize(deserializer)?.unwrap_or_default())
+}
+
+fn deserialize_optional_service_account_expiration<'de, D>(deserializer: D) -> Result<Option<OffsetDateTime>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let expiration = Option::<String>::deserialize(deserializer)?;
+    let Some(expiration) = expiration else {
+        return Ok(None);
+    };
+    let expiration = expiration.trim();
+    if expiration.is_empty() {
+        return Ok(None);
+    }
+
+    let expiration = OffsetDateTime::parse(expiration, &Rfc3339).map_err(D::Error::custom)?;
+    if expiration.unix_timestamp() == 0 {
+        return Ok(None);
+    }
+
+    Ok(Some(expiration))
+}
+
 /// SRSvcAccCreate - create operation
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SRSvcAccCreate {
@@ -503,6 +532,7 @@ pub struct SRSvcAccCreate {
     #[serde(rename = "secretKey")]
     pub secret_key: String,
 
+    #[serde(default, deserialize_with = "deserialize_vec_or_default")]
     pub groups: Vec<String>,
 
     pub claims: HashMap<String, serde_json::Value>,
@@ -516,7 +546,11 @@ pub struct SRSvcAccCreate {
 
     pub description: String,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        deserialize_with = "deserialize_optional_service_account_expiration"
+    )]
     pub expiration: Option<OffsetDateTime>,
 
     #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")]
@@ -1242,4 +1276,23 @@ mod tests {
 
         assert!(long_req.validate().is_ok());
     }
+
+    #[test]
+    fn test_sr_svc_acc_create_deserialize_empty_expiration_as_none() {
+        let payload = r#"{
+            "parent": "useralpha",
+            "accessKey": "svcalpha",
+            "secretKey": "svcAlphaSecret123",
+            "groups": [],
+            "claims": {},
+            "sessionPolicy": null,
+            "status": "on",
+            "name": "uploaderKey",
+            "description": "alpha upload key",
+            "expiration": "   "
+        }"#;
+
+        let svc: SRSvcAccCreate = serde_json::from_str(payload).unwrap();
+        assert!(svc.expiration.is_none());
+    }
 }

crates/mcp/Cargo.toml

@@ -57,5 +57,6 @@ tracing-subscriber.workspace = true
 # File handling and MIME type detection
 mime_guess = { workspace = true }
 
-[dev-dependencies]
+[lib]
+doctest = false
 # Testing framework and utilities

crates/metrics/Cargo.toml

@@ -25,6 +25,11 @@ keywords.workspace = true
 categories.workspace = true
 authors.workspace = true
 
+[features]
+default = []
+gpu = ["dep:nvml-wrapper"]
+full = ["gpu"]
+
 [dependencies]
 rustfs-ecstore = { workspace = true }
 rustfs-utils = { workspace = true }
@@ -33,6 +38,8 @@ sysinfo = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "rt", "time", "macros"] }
 tokio-util = { workspace = true }
 tracing = { workspace = true }
+thiserror = { workspace = true }
+nvml-wrapper = { workspace = true, optional = true }
 
 [lints]
 workspace = true

crates/metrics/src/collectors/global.rs

@@ -16,13 +16,31 @@
 //!
 //! This module provides the entry point for initializing all metrics collectors.
 //! The actual statistics collection functions are in `stats_collector.rs`.
+//!
+//! System monitoring collectors (migrated from `rustfs-obs::system`):
+//! - Process CPU metrics
+//! - Process memory metrics
+//! - Process disk I/O metrics
+//! - Process network I/O metrics
 
 use crate::collectors::stats_collector::{
     collect_bucket_replication_bandwidth_stats, collect_bucket_stats, collect_cluster_stats, collect_disk_stats,
     collect_process_stats,
 };
 use crate::collectors::{
-    collect_bucket_metrics, collect_bucket_replication_bandwidth_metrics, collect_cluster_metrics, collect_node_metrics,
+    // System monitoring collectors (migrated from rustfs-obs::system)
+    ProcessCpuStats,
+    ProcessDiskStats,
+    ProcessMemoryStats,
+    ProcessNetworkStats,
+    collect_bucket_metrics,
+    collect_bucket_replication_bandwidth_metrics,
+    collect_cluster_metrics,
+    collect_node_metrics,
+    collect_process_cpu_metrics,
+    collect_process_disk_metrics,
+    collect_process_memory_metrics,
+    collect_process_network_metrics,
     collect_resource_metrics,
 };
 use crate::constants::{
@@ -33,10 +51,19 @@ use crate::constants::{
 };
 use crate::format::report_metrics;
 use rustfs_utils::get_env_opt_u64;
+use std::borrow::Cow;
 use std::time::Duration;
+use sysinfo::{Pid, System};
 use tokio_util::sync::CancellationToken;
 use tracing::warn;
 
+/// Default interval for system monitoring metrics (15 seconds)
+const DEFAULT_SYSTEM_METRICS_INTERVAL: Duration = Duration::from_secs(15);
+/// Environment variable for system monitoring interval
+const ENV_SYSTEM_METRICS_INTERVAL: &str = "RUSTFS_METRICS_SYSTEM_INTERVAL_SEC";
+/// Legacy environment variable for system monitoring interval
+const LEGACY_SYSTEM_METRICS_INTERVAL: &str = "RUSTFS_OBS_METRICS_SYSTEM_INTERVAL_MS";
+
 /// Initialize all metrics collectors.
 ///
 /// This function spawns background tasks that periodically collect metrics
@@ -69,48 +96,36 @@ pub fn init_metrics_collectors(token: CancellationToken) {
     const LEGACY_RESOURCE_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL";
     const LEGACY_DEFAULT_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL";
 
-    fn parse_interval(msc: &str, legacy_msc: &str) -> Option<u64> {
-        get_env_opt_u64(msc)
-            .or_else(|| get_env_opt_u64(legacy_msc))
-            .filter(|&v| v > 0)
-    }
-
-    // Read intervals from environment or use defaults, ensuring zero is ignored.
-    let cluster_interval = parse_interval(ENV_CLUSTER_METRICS_INTERVAL, LEGACY_CLUSTER_INTERVAL)
-        .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
-        .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
-        .filter(|&v| v > 0)
-        .map(Duration::from_secs)
-        .unwrap_or(DEFAULT_CLUSTER_METRICS_INTERVAL);
-
-    let bucket_interval = parse_interval(ENV_BUCKET_METRICS_INTERVAL, LEGACY_BUCKET_INTERVAL)
-        .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
-        .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
-        .filter(|&v| v > 0)
-        .map(Duration::from_secs)
-        .unwrap_or(DEFAULT_BUCKET_METRICS_INTERVAL);
-
-    let bucket_replication_bandwidth_interval =
-        parse_interval(ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, LEGACY_REPLICATION_BANDWIDTH_INTERVAL)
+    /// Parse metrics interval from environment variables with fallback to default.
+    ///
+    /// Priority: primary_env > legacy_env > default_env > legacy_default > default_value
+    fn parse_metrics_interval(primary_env: &str, legacy_env: &str, default_interval: Duration) -> Duration {
+        get_env_opt_u64(primary_env)
+            .or_else(|| get_env_opt_u64(legacy_env))
             .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
             .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
             .filter(|&v| v > 0)
             .map(Duration::from_secs)
-            .unwrap_or(DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL);
+            .unwrap_or(default_interval)
+    }
 
-    let node_interval = parse_interval(ENV_NODE_METRICS_INTERVAL, LEGACY_NODE_INTERVAL)
-        .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
-        .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
-        .filter(|&v| v > 0)
-        .map(Duration::from_secs)
-        .unwrap_or(DEFAULT_NODE_METRICS_INTERVAL);
+    // Read intervals from environment or use defaults
+    let cluster_interval =
+        parse_metrics_interval(ENV_CLUSTER_METRICS_INTERVAL, LEGACY_CLUSTER_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL);
 
-    let resource_interval = parse_interval(ENV_RESOURCE_METRICS_INTERVAL, LEGACY_RESOURCE_INTERVAL)
-        .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
-        .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
-        .filter(|&v| v > 0)
-        .map(Duration::from_secs)
-        .unwrap_or(DEFAULT_RESOURCE_METRICS_INTERVAL);
+    let bucket_interval =
+        parse_metrics_interval(ENV_BUCKET_METRICS_INTERVAL, LEGACY_BUCKET_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL);
+
+    let bucket_replication_bandwidth_interval = parse_metrics_interval(
+        ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
+        LEGACY_REPLICATION_BANDWIDTH_INTERVAL,
+        DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
+    );
+
+    let node_interval = parse_metrics_interval(ENV_NODE_METRICS_INTERVAL, LEGACY_NODE_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL);
+
+    let resource_interval =
+        parse_metrics_interval(ENV_RESOURCE_METRICS_INTERVAL, LEGACY_RESOURCE_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL);
 
     // Spawn task for cluster metrics
     let token_clone = token.clone();
@@ -207,4 +222,132 @@ pub fn init_metrics_collectors(token: CancellationToken) {
             }
         }
     });
+
+    // Spawn task for system monitoring metrics (migrated from rustfs-obs::system)
+    let system_interval = get_env_opt_u64(ENV_SYSTEM_METRICS_INTERVAL)
+        .or_else(|| get_env_opt_u64(LEGACY_SYSTEM_METRICS_INTERVAL).map(|ms| ms / 1000)) // Convert ms to seconds
+        .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
+        .filter(|&v| v > 0)
+        .map(Duration::from_secs)
+        .unwrap_or(DEFAULT_SYSTEM_METRICS_INTERVAL);
+
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        // Get current process PID
+        let pid = match sysinfo::get_current_pid() {
+            Ok(p) => p,
+            Err(e) => {
+                warn!("Failed to get current PID for system monitoring: {}", e);
+                return;
+            }
+        };
+
+        let mut interval = tokio::time::interval(system_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    // Collect system monitoring metrics
+                    let metrics = collect_system_monitoring_metrics(pid);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("System monitoring metrics collection cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+}
+
+/// Collect all system monitoring metrics for a process.
+///
+/// This function collects CPU, memory, disk I/O, and network I/O metrics
+/// for the specified process PID.
+///
+/// # Arguments
+/// * `pid` - The process ID to monitor
+///
+/// # Returns
+/// A vector of Prometheus metrics for the process.
+fn collect_system_monitoring_metrics(pid: Pid) -> Vec<crate::format::PrometheusMetric> {
+    let mut metrics = Vec::new();
+    let mut system = System::new();
+
+    // Refresh process information
+    system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
+
+    if let Some(process) = system.process(pid) {
+        // Create labels with process attributes
+        let labels: Vec<(&'static str, Cow<'static, str>)> = vec![
+            ("process_pid", Cow::Owned(pid.as_u32().to_string())),
+            ("process_executable_name", Cow::Owned(process.name().to_string_lossy().to_string())),
+        ];
+
+        // Collect CPU metrics
+        let cpu_stats = ProcessCpuStats {
+            usage: process.cpu_usage() as f64,
+            utilization: process.cpu_usage() as f64, // Same as usage for single process
+        };
+        metrics.extend(collect_process_cpu_metrics(&cpu_stats, Some(&labels)));
+
+        // Collect memory metrics
+        let memory_stats = ProcessMemoryStats {
+            resident: process.memory(),
+            virtual_mem: process.virtual_memory(),
+        };
+        metrics.extend(collect_process_memory_metrics(&memory_stats, Some(&labels)));
+
+        // Collect disk I/O metrics
+        let disk_usage = process.disk_usage();
+        let disk_stats = ProcessDiskStats {
+            read_bytes: disk_usage.read_bytes,
+            written_bytes: disk_usage.written_bytes,
+        };
+        metrics.extend(collect_process_disk_metrics(&disk_stats, Some(&labels)));
+
+        // Collect network I/O metrics
+        // Note: sysinfo 0.38.x provides network info via Networks new type
+        // We use Networks::new_with_refreshed_list() to get network interfaces
+        let networks = sysinfo::Networks::new_with_refreshed_list();
+        let mut total_received = 0u64;
+        let mut total_transmitted = 0u64;
+        let mut per_interface = Vec::new();
+
+        for (interface_name, data) in networks.iter() {
+            let received = data.received();
+            let transmitted = data.transmitted();
+            total_received += received;
+            total_transmitted += transmitted;
+            per_interface.push((interface_name.to_string(), received, transmitted));
+        }
+
+        let network_stats = ProcessNetworkStats {
+            total_received,
+            total_transmitted,
+            per_interface,
+        };
+        metrics.extend(collect_process_network_metrics(&network_stats, Some(&labels)));
+
+        // Collect GPU metrics (if gpu feature is enabled)
+        #[cfg(feature = "gpu")]
+        {
+            use crate::collectors::{GpuCollector, collect_gpu_metrics};
+
+            match GpuCollector::new(pid) {
+                Ok(collector) => match collector.collect() {
+                    Ok(gpu_stats) => {
+                        metrics.extend(collect_gpu_metrics(&gpu_stats, &labels));
+                    }
+                    Err(e) => {
+                        warn!("GPU metrics collection failed: {}", e);
+                    }
+                },
+                Err(e) => {
+                    warn!("GPU collector initialization failed: {}", e);
+                }
+            }
+        }
+    }
+
+    metrics
 }

crates/metrics/src/collectors/mod.rs

@@ -82,6 +82,8 @@ mod scanner;
 mod stats_collector;
 mod system_cpu;
 mod system_drive;
+#[cfg(feature = "gpu")]
+mod system_gpu;
 mod system_memory;
 mod system_network;
 mod system_process;
@@ -104,8 +106,16 @@ pub use replication::{ReplicationStats, collect_replication_metrics};
 pub use request::{ApiRequestStats, collect_request_metrics};
 pub use resource::{ResourceStats, collect_resource_metrics};
 pub use scanner::{ScannerStats, collect_scanner_metrics};
-pub use system_cpu::{CpuStats, collect_cpu_metrics};
-pub use system_drive::{DriveCountStats, DriveDetailedStats, collect_drive_count_metrics, collect_drive_detailed_metrics};
-pub use system_memory::{MemoryStats, collect_memory_metrics};
-pub use system_network::{NetworkStats, collect_network_metrics};
-pub use system_process::{ProcessStats, collect_process_metrics};
+pub use system_cpu::{CpuStats, ProcessCpuStats, collect_cpu_metrics, collect_process_cpu_metrics};
+pub use system_drive::{
+    DriveCountStats, DriveDetailedStats, ProcessDiskStats, collect_drive_count_metrics, collect_drive_detailed_metrics,
+    collect_process_disk_metrics,
+};
+#[cfg(feature = "gpu")]
+pub use system_gpu::{GpuCollector, GpuError, GpuStats, collect_gpu_metrics};
+pub use system_memory::{MemoryStats, ProcessMemoryStats, collect_memory_metrics, collect_process_memory_metrics};
+pub use system_network::{NetworkStats, ProcessNetworkStats, collect_network_metrics, collect_process_network_metrics};
+pub use system_process::{
+    ProcessAttributeError, ProcessAttributes, ProcessStats, ProcessStatusType, collect_process_attributes,
+    collect_process_metrics,
+};

crates/metrics/src/collectors/system_cpu.rs

@@ -16,16 +16,18 @@
 
 //! System CPU metrics collector.
 //!
-//! Collects CPU-related metrics including load average, idle time,
-//! I/O wait, and CPU usage percentages.
+//! Collects CPU metrics including load average, CPU time distribution,
+//! and process-level CPU usage.
 //!
-//! This collector reuses the metric descriptors defined in `metrics_type::system_cpu`
-//! to avoid duplication of metric names, types, and help text.
+//! This module provides both system-level and process-level CPU metrics,
+//! with process-level metrics migrated from `rustfs-obs::system`.
 
 use crate::format::PrometheusMetric;
 use crate::metrics_type::system_cpu::*;
+use crate::metrics_type::system_process::{PROCESS_CPU_USAGE_MD, PROCESS_CPU_UTILIZATION_MD};
+use std::borrow::Cow;
 
-/// CPU statistics for a node.
+/// System CPU statistics.
 #[derive(Debug, Clone, Default)]
 pub struct CpuStats {
     /// Average CPU idle time (percentage, 0-100)
@@ -46,6 +48,17 @@ pub struct CpuStats {
     pub user: f64,
 }
 
+/// Process CPU statistics.
+///
+/// Contains CPU usage metrics for a specific process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessCpuStats {
+    /// CPU usage percentage (0-100)
+    pub usage: f64,
+    /// CPU utilization percentage (considering multiple cores, can exceed 100)
+    pub utilization: f64,
+}
+
 /// Collects CPU metrics from the given stats.
 ///
 /// Uses the metric descriptors from `metrics_type::system_cpu` module.
@@ -63,6 +76,29 @@ pub fn collect_cpu_metrics(stats: &CpuStats) -> Vec<PrometheusMetric> {
     ]
 }
 
+/// Collects process CPU metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::system_process` module.
+/// Returns a vector of Prometheus metrics for process CPU statistics.
+///
+/// # Arguments
+///
+/// * `stats` - Process CPU statistics
+/// * `labels` - Optional additional labels (e.g., process attributes)
+pub fn collect_process_cpu_metrics(
+    stats: &ProcessCpuStats,
+    labels: Option<&[(&'static str, Cow<'static, str>)]>,
+) -> Vec<PrometheusMetric> {
+    let mut usage_metric = PrometheusMetric::from_descriptor(&PROCESS_CPU_USAGE_MD, stats.usage);
+    let mut utilization_metric = PrometheusMetric::from_descriptor(&PROCESS_CPU_UTILIZATION_MD, stats.utilization);
+
+    if let Some(l) = labels {
+        usage_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        utilization_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+    }
+
+    vec![usage_metric, utilization_metric]
+}
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -101,4 +137,48 @@ mod tests {
             assert!(metric.labels.is_empty());
         }
     }
+
+    #[test]
+    fn test_collect_process_cpu_metrics() {
+        let stats = ProcessCpuStats {
+            usage: 45.5,
+            utilization: 182.0, // 4 cores at ~45% each
+        };
+
+        let metrics = collect_process_cpu_metrics(&stats, None);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 2);
+
+        // Verify usage metric
+        let usage_metric = metrics.iter().find(|m| m.name.contains("cpu_usage"));
+        assert!(usage_metric.is_some());
+        assert_eq!(usage_metric.map(|m| m.value), Some(45.5));
+
+        // Verify utilization metric
+        let util_metric = metrics.iter().find(|m| m.name.contains("cpu_utilization"));
+        assert!(util_metric.is_some());
+        assert_eq!(util_metric.map(|m| m.value), Some(182.0));
+    }
+
+    #[test]
+    fn test_collect_process_cpu_metrics_with_labels() {
+        let stats = ProcessCpuStats {
+            usage: 25.0,
+            utilization: 100.0,
+        };
+
+        let labels = vec![
+            ("process_pid", Cow::Borrowed("12345")),
+            ("process_executable_name", Cow::Borrowed("rustfs")),
+        ];
+
+        let metrics = collect_process_cpu_metrics(&stats, Some(&labels));
+        assert_eq!(metrics.len(), 2);
+
+        // All metrics should have the labels
+        for metric in &metrics {
+            assert_eq!(metric.labels.len(), 2);
+        }
+    }
 }

crates/metrics/src/collectors/system_drive.rs

@@ -18,9 +18,14 @@
 //!
 //! Collects detailed drive/disk metrics including capacity, I/O statistics,
 //! error counts, and health status.
+//!
+//! This module provides both system-level and process-level disk metrics,
+//! with process-level metrics migrated from `rustfs-obs::system`.
 
 use crate::format::PrometheusMetric;
 use crate::metrics_type::system_drive::*;
+use crate::metrics_type::system_process::PROCESS_DISK_IO_MD;
+use std::borrow::Cow;
 
 /// Detailed drive statistics for a single drive.
 #[derive(Debug, Clone, Default)]
@@ -163,6 +168,44 @@ pub fn collect_drive_count_metrics(stats: &DriveCountStats) -> Vec<PrometheusMet
     ]
 }
 
+/// Process disk I/O statistics.
+///
+/// Contains disk I/O metrics for a specific process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessDiskStats {
+    /// Bytes read from disk
+    pub read_bytes: u64,
+    /// Bytes written to disk
+    pub written_bytes: u64,
+}
+
+/// Collects process disk I/O metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for process disk I/O statistics.
+/// Each metric includes a `direction` label ("read" or "write").
+///
+/// # Arguments
+///
+/// * `stats` - Process disk I/O statistics
+/// * `labels` - Optional additional labels (e.g., process attributes)
+pub fn collect_process_disk_metrics(
+    stats: &ProcessDiskStats,
+    labels: Option<&[(&'static str, Cow<'static, str>)]>,
+) -> Vec<PrometheusMetric> {
+    let mut read_metric = PrometheusMetric::from_descriptor(&PROCESS_DISK_IO_MD, stats.read_bytes as f64);
+    let mut write_metric = PrometheusMetric::from_descriptor(&PROCESS_DISK_IO_MD, stats.written_bytes as f64);
+
+    read_metric.labels.push(("direction", Cow::Borrowed("read")));
+    write_metric.labels.push(("direction", Cow::Borrowed("write")));
+
+    if let Some(l) = labels {
+        read_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        write_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+    }
+
+    vec![read_metric, write_metric]
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/metrics/src/collectors/system_gpu.rs

@@ -0,0 +1,199 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! System GPU metrics collector.
+//!
+//! Collects GPU memory usage metrics using NVML library.
+//! This module is only available when `gpu` feature is enabled.
+//!
+//! # Example
+//!
+//! ```ignore
+//! use rustfs_metrics::collectors::{GpuCollector, collect_gpu_metrics};
+//! use sysinfo::Pid;
+//!
+//! let pid = sysinfo::get_current_pid().unwrap();
+//! let collector = GpuCollector::new(pid)?;
+//! let stats = collector.collect()?;
+//! let metrics = collect_gpu_metrics(&stats, &labels);
+//! ```
+
+use crate::format::PrometheusMetric;
+
+use crate::metrics_type::system_gpu::PROCESS_GPU_MEMORY_USAGE_MD;
+
+use nvml_wrapper::Nvml;
+
+use nvml_wrapper::enums::device::UsedGpuMemory;
+
+use sysinfo::Pid;
+
+use std::borrow::Cow;
+
+use thiserror::Error;
+
+use tracing::warn;
+
+/// GPU statistics.
+///
+/// Contains GPU memory usage metrics for the monitored process.
+
+#[derive(Debug, Clone, Default)]
+pub struct GpuStats {
+    /// GPU memory usage in bytes
+    pub memory_usage: u64,
+}
+
+/// GPU collector error types.
+
+#[derive(Debug, Error)]
+pub enum GpuError {
+    /// GPU initialization failed
+    #[error("GPU initialization failed: {0}")]
+    InitError(String),
+
+    /// GPU device access error
+    #[error("GPU device error: {0}")]
+    DeviceError(String),
+
+    /// Process not found in GPU process list
+    #[error("Process not found in GPU process list")]
+    ProcessNotFound,
+}
+
+/// GPU metrics collector.
+///
+/// Collects GPU memory usage metrics for a specific process using NVML.
+pub struct GpuCollector {
+    /// NVML instance for GPU access
+    nvml: Nvml,
+    /// Process ID to monitor
+    pid: Pid,
+}
+
+impl GpuCollector {
+    /// Creates a new GPU collector for the specified process.
+    ///
+    /// # Arguments
+    ///
+    /// * `pid` - The process ID to monitor
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if NVML initialization fails.
+    ///
+    /// # Example
+    ///
+    /// ```ignore
+    /// use rustfs_metrics::collectors::GpuCollector;
+    /// use sysinfo::Pid;
+    ///
+    /// let pid = sysinfo::get_current_pid().unwrap();
+    /// let collector = GpuCollector::new(pid)?;
+    /// ```
+    pub fn new(pid: Pid) -> Result<Self, GpuError> {
+        let nvml = Nvml::init().map_err(|e| GpuError::InitError(e.to_string()))?;
+        Ok(GpuCollector { nvml, pid })
+    }
+
+    /// Collects GPU metrics for the monitored process.
+    ///
+    /// Returns GPU memory usage statistics for the process.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if GPU device access fails.
+    ///
+    /// # Example
+    ///
+    /// ```ignore
+    /// let stats = collector.collect()?;
+    /// println!("GPU memory usage: {} bytes", stats.memory_usage);
+    /// ```
+    pub fn collect(&self) -> Result<GpuStats, GpuError> {
+        if let Ok(device) = self.nvml.device_by_index(0) {
+            if let Ok(gpu_stats) = device.running_compute_processes() {
+                for stat in gpu_stats.iter() {
+                    if stat.pid == self.pid.as_u32() {
+                        let memory_used = match stat.used_gpu_memory {
+                            UsedGpuMemory::Used(bytes) => bytes,
+                            UsedGpuMemory::Unavailable => 0,
+                        };
+                        return Ok(GpuStats {
+                            memory_usage: memory_used,
+                        });
+                    }
+                }
+            } else {
+                warn!("Could not get GPU stats, recording 0 for GPU memory usage");
+            }
+        } else {
+            return Err(GpuError::DeviceError("No GPU device found".to_string()));
+        }
+
+        // Process not found in GPU process list, return 0 usage
+        Ok(GpuStats { memory_usage: 0 })
+    }
+}
+
+/// Converts GPU stats to Prometheus metrics.
+///
+/// # Arguments
+///
+/// * `stats` - GPU statistics to convert
+/// * `labels` - Metric labels (typically from ProcessAttributes)
+///
+/// # Returns
+///
+/// A vector of Prometheus metrics.
+///
+/// # Example
+///
+/// ```ignore
+/// use rustfs_metrics::collectors::{GpuStats, collect_gpu_metrics};
+///
+/// let stats = GpuStats { memory_usage: 1024 };
+/// let labels = vec![("process_pid", Cow::Borrowed("1234"))];
+/// let metrics = collect_gpu_metrics(&stats, &labels);
+/// ```
+pub fn collect_gpu_metrics(stats: &GpuStats, labels: &[(&'static str, Cow<'static, str>)]) -> Vec<PrometheusMetric> {
+    let mut metric = PrometheusMetric::from_descriptor(&PROCESS_GPU_MEMORY_USAGE_MD, stats.memory_usage as f64);
+    metric.labels.extend(labels.iter().map(|(k, v)| (*k, v.clone())));
+    vec![metric]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_gpu_stats_default() {
+        let stats = GpuStats::default();
+        assert_eq!(stats.memory_usage, 0);
+    }
+
+    #[test]
+    fn test_gpu_error_display() {
+        let err = GpuError::InitError("test error".to_string());
+        assert!(err.to_string().contains("test error"));
+
+        let err = GpuError::DeviceError("device error".to_string());
+        assert!(err.to_string().contains("device error"));
+
+        let err = GpuError::ProcessNotFound;
+        assert!(err.to_string().contains("Process not found"));
+    }
+}

crates/metrics/src/collectors/system_memory.rs

@@ -19,13 +19,15 @@
 //! Collects memory-related metrics including total, used, free,
 //! buffers, cache, shared, and available memory.
 //!
-//! This collector reuses the metric descriptors defined in `metrics_type::system_memory`
-//! to avoid duplication of metric names, types, and help text.
+//! This module provides both system-level and process-level memory metrics,
+//! with process-level metrics migrated from `rustfs-obs::system`.
 
 use crate::format::PrometheusMetric;
 use crate::metrics_type::system_memory::*;
+use crate::metrics_type::system_process::{PROCESS_RESIDENT_MEMORY_BYTES_MD, PROCESS_VIRTUAL_MEMORY_BYTES_MD};
+use std::borrow::Cow;
 
-/// Memory statistics for a node.
+/// System memory statistics.
 #[derive(Debug, Clone, Default)]
 pub struct MemoryStats {
     /// Total memory in bytes
@@ -46,6 +48,17 @@ pub struct MemoryStats {
     pub available: u64,
 }
 
+/// Process memory statistics.
+///
+/// Contains memory usage metrics for a specific process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessMemoryStats {
+    /// Resident memory size in bytes
+    pub resident: u64,
+    /// Virtual memory size in bytes
+    pub virtual_mem: u64,
+}
+
 /// Collects memory metrics from the given stats.
 ///
 /// Uses the metric descriptors from `metrics_type::system_memory` module.
@@ -63,6 +76,30 @@ pub fn collect_memory_metrics(stats: &MemoryStats) -> Vec<PrometheusMetric> {
     ]
 }
 
+/// Collects process memory metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::system_process` module.
+/// Returns a vector of Prometheus metrics for process memory statistics.
+///
+/// # Arguments
+///
+/// * `stats` - Process memory statistics
+/// * `labels` - Optional additional labels (e.g., process attributes)
+pub fn collect_process_memory_metrics(
+    stats: &ProcessMemoryStats,
+    labels: Option<&[(&'static str, Cow<'static, str>)]>,
+) -> Vec<PrometheusMetric> {
+    let mut resident_metric = PrometheusMetric::from_descriptor(&PROCESS_RESIDENT_MEMORY_BYTES_MD, stats.resident as f64);
+    let mut virtual_metric = PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_BYTES_MD, stats.virtual_mem as f64);
+
+    if let Some(l) = labels {
+        resident_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        virtual_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+    }
+
+    vec![resident_metric, virtual_metric]
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -99,4 +136,34 @@ mod tests {
             assert!(metric.labels.is_empty());
         }
     }
+
+    #[test]
+    fn test_collect_process_memory_metrics() {
+        let stats = ProcessMemoryStats {
+            resident: 512 * 1024 * 1024,         // 512 MB
+            virtual_mem: 2 * 1024 * 1024 * 1024, // 2 GB
+        };
+
+        let metrics = collect_process_memory_metrics(&stats, None);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 2);
+    }
+
+    #[test]
+    fn test_collect_process_memory_metrics_with_labels() {
+        let stats = ProcessMemoryStats {
+            resident: 256 * 1024 * 1024,
+            virtual_mem: 1024 * 1024 * 1024,
+        };
+
+        let labels = vec![("process_pid", Cow::Borrowed("12345"))];
+
+        let metrics = collect_process_memory_metrics(&stats, Some(&labels));
+        assert_eq!(metrics.len(), 2);
+
+        for metric in &metrics {
+            assert_eq!(metric.labels.len(), 1);
+        }
+    }
 }

crates/metrics/src/collectors/system_network.rs

@@ -19,11 +19,13 @@
 //! Collects internode network metrics including errors, dial times,
 //! and bytes sent/received.
 //!
-//! This collector reuses the metric descriptors defined in `metrics_type::system_network`
-//! to avoid duplication of metric names, types, and help text.
+//! This module provides both system-level and process-level network metrics,
+//! with process-level metrics migrated from `rustfs-obs::system`.
 
 use crate::format::PrometheusMetric;
 use crate::metrics_type::system_network::*;
+use crate::metrics_type::system_process::{PROCESS_NETWORK_IO_MD, PROCESS_NETWORK_IO_PER_INTERFACE_MD};
+use std::borrow::Cow;
 
 /// Network statistics for internode communication.
 #[derive(Debug, Clone, Default)]
@@ -40,6 +42,19 @@ pub struct NetworkStats {
     pub internode_recv_bytes_total: u64,
 }
 
+/// Process network I/O statistics.
+///
+/// Contains network I/O metrics for a specific process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessNetworkStats {
+    /// Total bytes received
+    pub total_received: u64,
+    /// Total bytes transmitted
+    pub total_transmitted: u64,
+    /// Per-interface statistics: (interface_name, received_bytes, transmitted_bytes)
+    pub per_interface: Vec<(String, u64, u64)>,
+}
+
 /// Collects network metrics from the given stats.
 ///
 /// Uses the metric descriptors from `metrics_type::system_network` module.
@@ -54,6 +69,60 @@ pub fn collect_network_metrics(stats: &NetworkStats) -> Vec<PrometheusMetric> {
     ]
 }
 
+/// Collects process network I/O metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for process network I/O statistics.
+/// Each metric includes a `direction` label ("received" or "transmitted").
+/// Per-interface metrics also include an `interface` label.
+///
+/// # Arguments
+///
+/// * `stats` - Process network I/O statistics
+/// * `labels` - Optional additional labels (e.g., process attributes)
+pub fn collect_process_network_metrics(
+    stats: &ProcessNetworkStats,
+    labels: Option<&[(&'static str, Cow<'static, str>)]>,
+) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(2 + stats.per_interface.len() * 2);
+
+    // Total network I/O
+    let mut received_metric = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_MD, stats.total_received as f64);
+    let mut transmitted_metric = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_MD, stats.total_transmitted as f64);
+
+    received_metric.labels.push(("direction", Cow::Borrowed("received")));
+    transmitted_metric.labels.push(("direction", Cow::Borrowed("transmitted")));
+
+    if let Some(l) = labels {
+        received_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        transmitted_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+    }
+
+    metrics.push(received_metric);
+    metrics.push(transmitted_metric);
+
+    // Per-interface network I/O
+    for (interface, received, transmitted) in &stats.per_interface {
+        let mut iface_received = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_PER_INTERFACE_MD, *received as f64);
+        let mut iface_transmitted = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_PER_INTERFACE_MD, *transmitted as f64);
+
+        iface_received.labels.push(("interface", Cow::Owned(interface.clone())));
+        iface_received.labels.push(("direction", Cow::Borrowed("received")));
+
+        iface_transmitted.labels.push(("interface", Cow::Owned(interface.clone())));
+        iface_transmitted.labels.push(("direction", Cow::Borrowed("transmitted")));
+
+        if let Some(l) = labels {
+            iface_received.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+            iface_transmitted.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        }
+
+        metrics.push(iface_received);
+        metrics.push(iface_transmitted);
+    }
+
+    metrics
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/metrics/src/collectors/system_process.rs

@@ -18,9 +18,130 @@
 //!
 //! Collects process-level metrics including file descriptors, memory,
 //! syscalls, and runtime statistics.
+//!
+//! This module also provides process attribute collection for use as
+//! metric labels, migrated from `rustfs-obs::system`.
 
 use crate::format::PrometheusMetric;
 use crate::metrics_type::system_process::*;
+use std::borrow::Cow;
+use sysinfo::{Pid, ProcessStatus, System};
+
+/// Process attributes used as metric labels.
+///
+/// Contains identifying information about the process being monitored.
+#[derive(Debug, Clone)]
+pub struct ProcessAttributes {
+    /// Process ID
+    pub pid: u32,
+    /// Executable name (e.g., "rustfs")
+    pub executable_name: String,
+    /// Full path to the executable
+    pub executable_path: String,
+    /// Full command line with arguments
+    pub command: String,
+}
+
+impl ProcessAttributes {
+    /// Creates a new instance by reading from the current process.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the current process PID cannot be determined
+    /// or if process information cannot be retrieved.
+    pub fn current() -> Result<Self, ProcessAttributeError> {
+        let pid = sysinfo::get_current_pid().map_err(|e| ProcessAttributeError::PidError(e.to_string()))?;
+        Self::from_pid(pid)
+    }
+
+    /// Creates a new instance for a specific PID.
+    ///
+    /// # Arguments
+    ///
+    /// * `pid` - The process ID to query
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the process does not exist or information
+    /// cannot be retrieved.
+    pub fn from_pid(pid: Pid) -> Result<Self, ProcessAttributeError> {
+        let mut system = System::new();
+        system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
+
+        let process = system
+            .process(pid)
+            .ok_or_else(|| ProcessAttributeError::ProcessNotFound(pid.as_u32()))?;
+
+        Ok(ProcessAttributes {
+            pid: pid.as_u32(),
+            executable_name: process.name().to_string_lossy().to_string(),
+            executable_path: process.exe().map(|p| p.to_string_lossy().to_string()).unwrap_or_default(),
+            command: process
+                .cmd()
+                .iter()
+                .map(|s| s.to_string_lossy().into_owned())
+                .collect::<Vec<_>>()
+                .join(" "),
+        })
+    }
+
+    /// Converts attributes to Prometheus metric labels.
+    pub fn to_labels(&self) -> Vec<(&'static str, Cow<'static, str>)> {
+        vec![
+            ("process_pid", Cow::Owned(self.pid.to_string())),
+            ("process_executable_name", Cow::Owned(self.executable_name.clone())),
+            ("process_executable_path", Cow::Owned(self.executable_path.clone())),
+            ("process_command", Cow::Owned(self.command.clone())),
+        ]
+    }
+}
+
+/// Errors that can occur when collecting process attributes.
+#[derive(Debug, Clone)]
+pub enum ProcessAttributeError {
+    /// Failed to get current process PID
+    PidError(String),
+    /// Process not found
+    ProcessNotFound(u32),
+}
+
+impl std::fmt::Display for ProcessAttributeError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::PidError(e) => write!(f, "Failed to get current PID: {}", e),
+            Self::ProcessNotFound(pid) => write!(f, "Process not found: {}", pid),
+        }
+    }
+}
+
+impl std::error::Error for ProcessAttributeError {}
+
+/// Process status enumeration.
+///
+/// Maps `sysinfo::ProcessStatus` to a simpler representation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum ProcessStatusType {
+    /// Process is currently running
+    Running = 0,
+    /// Process is sleeping (waiting for I/O or event)
+    Sleeping = 1,
+    /// Process is a zombie (terminated but not reaped)
+    Zombie = 2,
+    /// Process is in some other state
+    #[default]
+    Other = 3,
+}
+
+impl From<ProcessStatus> for ProcessStatusType {
+    fn from(status: ProcessStatus) -> Self {
+        match status {
+            ProcessStatus::Run => ProcessStatusType::Running,
+            ProcessStatus::Sleep => ProcessStatusType::Sleeping,
+            ProcessStatus::Zombie => ProcessStatusType::Zombie,
+            _ => ProcessStatusType::Other,
+        }
+    }
+}
 
 /// Process statistics for the RustFS server process.
 #[derive(Debug, Clone, Default)]
@@ -59,13 +180,17 @@ pub struct ProcessStats {
     pub virtual_memory_bytes: u64,
     /// Maximum virtual memory size in bytes
     pub virtual_memory_max_bytes: u64,
+    /// Process status
+    pub status: ProcessStatusType,
+    /// Process status value (numeric)
+    pub status_value: i64,
 }
 
 /// Collects process metrics from the given stats.
 ///
 /// Returns a vector of Prometheus metrics for process statistics.
 pub fn collect_process_metrics(stats: &ProcessStats) -> Vec<PrometheusMetric> {
-    vec![
+    let mut metrics = vec![
         PrometheusMetric::from_descriptor(&PROCESS_LOCKS_READ_TOTAL_MD, stats.locks_read_total as f64),
         PrometheusMetric::from_descriptor(&PROCESS_LOCKS_WRITE_TOTAL_MD, stats.locks_write_total as f64),
         PrometheusMetric::from_descriptor(&PROCESS_CPU_TOTAL_SECONDS_MD, stats.cpu_total_seconds),
@@ -83,7 +208,23 @@ pub fn collect_process_metrics(stats: &ProcessStats) -> Vec<PrometheusMetric> {
         PrometheusMetric::from_descriptor(&PROCESS_RESIDENT_MEMORY_BYTES_MD, stats.resident_memory_bytes as f64),
         PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_BYTES_MD, stats.virtual_memory_bytes as f64),
         PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD, stats.virtual_memory_max_bytes as f64),
-    ]
+    ];
+
+    // Add process status metric
+    let mut status_metric = PrometheusMetric::from_descriptor(&PROCESS_STATUS_MD, stats.status_value as f64);
+    status_metric
+        .labels
+        .push(("status", Cow::Owned(format!("{:?}", stats.status))));
+    metrics.push(status_metric);
+
+    metrics
+}
+
+/// Collects process attributes for the current process.
+///
+/// This is a convenience function that wraps `ProcessAttributes::current()`.
+pub fn collect_process_attributes() -> Result<ProcessAttributes, ProcessAttributeError> {
+    ProcessAttributes::current()
 }
 
 #[cfg(test)]
@@ -111,12 +252,15 @@ mod tests {
             resident_memory_bytes: 1024 * 1024 * 512,
             virtual_memory_bytes: 1024 * 1024 * 1024,
             virtual_memory_max_bytes: 1024 * 1024 * 2048,
+            status: ProcessStatusType::Running,
+            status_value: 0,
         };
 
         let metrics = collect_process_metrics(&stats);
         report_metrics(&metrics);
 
-        assert_eq!(metrics.len(), 17);
+        // 17 original metrics + 1 status metric = 18
+        assert_eq!(metrics.len(), 18);
 
         // Verify uptime
         let uptime_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name();
@@ -129,6 +273,12 @@ mod tests {
         let fd_open = metrics.iter().find(|m| m.name == fd_open_name);
         assert!(fd_open.is_some());
         assert_eq!(fd_open.map(|m| m.value), Some(1500.0));
+
+        // Verify status metric
+        let status_name = PROCESS_STATUS_MD.get_full_metric_name();
+        let status_metric = metrics.iter().find(|m| m.name == status_name);
+        assert!(status_metric.is_some());
+        assert_eq!(status_metric.map(|m| m.value), Some(0.0));
     }
 
     #[test]
@@ -136,10 +286,40 @@ mod tests {
         let stats = ProcessStats::default();
         let metrics = collect_process_metrics(&stats);
 
-        assert_eq!(metrics.len(), 17);
-        for metric in &metrics {
-            assert_eq!(metric.value, 0.0);
-            assert!(metric.labels.is_empty());
-        }
+        // 17 original metrics + 1 status metric = 18
+        assert_eq!(metrics.len(), 18);
+    }
+
+    #[test]
+    fn test_process_attributes_current() {
+        // This test should succeed as we're querying the current process
+        let result = collect_process_attributes();
+        assert!(result.is_ok());
+
+        let attrs = result.unwrap();
+        assert!(attrs.pid > 0);
+        assert!(!attrs.executable_name.is_empty());
+    }
+
+    #[test]
+    fn test_process_status_conversion() {
+        assert_eq!(ProcessStatusType::from(ProcessStatus::Run), ProcessStatusType::Running);
+        assert_eq!(ProcessStatusType::from(ProcessStatus::Sleep), ProcessStatusType::Sleeping);
+        assert_eq!(ProcessStatusType::from(ProcessStatus::Zombie), ProcessStatusType::Zombie);
+    }
+
+    #[test]
+    fn test_process_attributes_to_labels() {
+        let attrs = ProcessAttributes {
+            pid: 12345,
+            executable_name: "rustfs".to_string(),
+            executable_path: "/usr/bin/rustfs".to_string(),
+            command: "rustfs server /data".to_string(),
+        };
+
+        let labels = attrs.to_labels();
+        assert_eq!(labels.len(), 4);
+        assert_eq!(labels[0].0, "process_pid");
+        assert_eq!(labels[0].1, "12345");
     }
 }

crates/metrics/src/metrics_type/entry/metric_name.rs

@@ -350,6 +350,22 @@ pub enum MetricName {
     ProcessVirtualMemoryBytes,
     ProcessVirtualMemoryMaxBytes,
 
+    // Process-level system monitoring metrics (migrated from rustfs-obs::system)
+    /// Process CPU usage percentage (0-100)
+    ProcessCPUUsage,
+    /// Process CPU utilization percentage (considering multiple cores)
+    ProcessCPUUtilization,
+    /// Process disk I/O bytes
+    ProcessDiskIO,
+    /// Process network I/O bytes
+    ProcessNetworkIO,
+    /// Process network I/O bytes per interface
+    ProcessNetworkIOPerInterface,
+    /// Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other)
+    ProcessStatus,
+    /// Process GPU memory usage in bytes
+    ProcessGpuMemoryUsage,
+
     // Custom metrics
     Custom(String),
 }
@@ -672,6 +688,15 @@ impl MetricName {
             Self::ProcessVirtualMemoryBytes => "virtual_memory_bytes".to_string(),
             Self::ProcessVirtualMemoryMaxBytes => "virtual_memory_max_bytes".to_string(),
 
+            // Process-level system monitoring metrics (migrated from rustfs-obs::system)
+            Self::ProcessCPUUsage => "cpu_usage".to_string(),
+            Self::ProcessCPUUtilization => "cpu_utilization".to_string(),
+            Self::ProcessDiskIO => "disk_io".to_string(),
+            Self::ProcessNetworkIO => "network_io".to_string(),
+            Self::ProcessNetworkIOPerInterface => "network_io_per_interface".to_string(),
+            Self::ProcessGpuMemoryUsage => "gpu_memory_usage".to_string(),
+            Self::ProcessStatus => "status".to_string(),
+
             Self::Custom(name) => name.clone(),
         }
     }

crates/metrics/src/metrics_type/entry/subsystem.rs

@@ -32,6 +32,7 @@ pub enum MetricSubsystem {
     SystemCpu,
     SystemProcess,
 
+    SystemGpu,
     // debug related subsystems
     DebugGo,
 
@@ -74,6 +75,7 @@ impl MetricSubsystem {
             Self::SystemProcess => "/system/process",
 
             // debug related subsystems
+            Self::SystemGpu => "/system/gpu",
             Self::DebugGo => "/debug/go",
 
             // cluster related subsystems
@@ -120,6 +122,7 @@ impl MetricSubsystem {
             "/system/cpu" => Self::SystemCpu,
             "/system/process" => Self::SystemProcess,
 
+            "/system/gpu" => Self::SystemGpu,
             // Debug related subsystems
             "/debug/go" => Self::DebugGo,
 
@@ -181,6 +184,7 @@ pub mod subsystems {
     pub const API_REQUESTS: MetricSubsystem = MetricSubsystem::ApiRequests;
     pub const BUCKET_API: MetricSubsystem = MetricSubsystem::BucketApi;
     pub const BUCKET_REPLICATION: MetricSubsystem = MetricSubsystem::BucketReplication;
+    pub const SYSTEM_GPU: MetricSubsystem = MetricSubsystem::SystemGpu;
     pub const SYSTEM_NETWORK_INTERNODE: MetricSubsystem = MetricSubsystem::SystemNetworkInternode;
     pub const SYSTEM_DRIVE: MetricSubsystem = MetricSubsystem::SystemDrive;
     pub const SYSTEM_MEMORY: MetricSubsystem = MetricSubsystem::SystemMemory;

crates/metrics/src/metrics_type/mod.rs

@@ -33,6 +33,7 @@ pub mod request;
 pub mod scanner;
 pub mod system_cpu;
 pub mod system_drive;
+pub mod system_gpu;
 pub mod system_memory;
 pub mod system_network;
 pub mod system_process;

crates/metrics/src/metrics_type/system_gpu.rs

@@ -0,0 +1,35 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! GPU-related metric descriptors.
+//!
+//! This module defines metric descriptors for GPU monitoring,
+//! including GPU memory usage metrics.
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Process GPU memory usage metric descriptor.
+///
+/// Records the amount of physical GPU memory in use by the process.
+pub static PROCESS_GPU_MEMORY_USAGE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessGpuMemoryUsage,
+        "The amount of physical GPU memory in use",
+        &[],
+        subsystems::SYSTEM_GPU,
+    )
+});

crates/metrics/src/metrics_type/system_process.rs

@@ -186,3 +186,67 @@ pub static PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD: LazyLock<MetricDescriptor> = Laz
         subsystems::SYSTEM_PROCESS,
     )
 });
+
+// ============================================================================
+// Process-level system monitoring metrics (migrated from rustfs-obs::system)
+// ============================================================================
+
+/// Process CPU usage percentage (0-100)
+pub static PROCESS_CPU_USAGE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessCPUUsage,
+        "The percentage of CPU in use by the process",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Process CPU utilization percentage (considering multiple cores)
+pub static PROCESS_CPU_UTILIZATION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessCPUUtilization,
+        "The amount of CPU in use by the process (considering multiple cores)",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Process disk I/O bytes
+pub static PROCESS_DISK_IO_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessDiskIO,
+        "Disk bytes transferred by the process",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Process network I/O bytes
+pub static PROCESS_NETWORK_IO_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessNetworkIO,
+        "Network bytes transferred by the process",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Process network I/O bytes per interface
+pub static PROCESS_NETWORK_IO_PER_INTERFACE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessNetworkIOPerInterface,
+        "Network bytes transferred by the process (per interface)",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other)
+pub static PROCESS_STATUS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessStatus,
+        "Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other)",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});

crates/notify/Cargo.toml

@@ -64,3 +64,6 @@ serde_json = { workspace = true }
 
 [lints]
 workspace = true
+
+[lib]
+doctest = false

crates/notify/src/registry.rs

@@ -293,6 +293,12 @@ impl TargetRegistry {
                 }
             }
 
+            if &new_config == config {
+                info!("Notification target configuration unchanged, skip persisting server config");
+                info!(count = successful_targets.len(), "All target processing completed");
+                return Ok(successful_targets);
+            }
+
             let store = match rustfs_ecstore::global::new_object_layer_fn() {
                 Some(s) => s,
                 None => {

crates/obs/Cargo.toml

@@ -28,14 +28,9 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/"
 [lints]
 workspace = true
 
-[features]
-default = []
-gpu = ["dep:nvml-wrapper"]
-full = ["gpu"]
-
 [dependencies]
 rustfs-config = { workspace = true, features = ["constants", "observability"] }
-rustfs-utils = { workspace = true, features = ["ip", "path"] }
+rustfs-utils = { workspace = true, features = ["ip"] }
 flate2 = { workspace = true }
 glob = { workspace = true }
 jiff = { workspace = true }
@@ -44,7 +39,6 @@ crossbeam-channel = { workspace = true }
 crossbeam-deque = { workspace = true }
 crossbeam-utils = { workspace = true }
 num_cpus = { workspace = true }
-nvml-wrapper = { workspace = true, optional = true }
 opentelemetry = { workspace = true }
 opentelemetry-appender-tracing = { workspace = true, features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] }
 opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
@@ -58,7 +52,6 @@ tracing-error = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
 tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
-sysinfo = { workspace = true }
 thiserror = { workspace = true }
 zstd = { workspace = true, features = ["zstdmt"] }
 
@@ -70,4 +63,3 @@ pyroscope = { workspace = true, features = ["backend-pprof-rs"] }
 tokio = { workspace = true, features = ["full"] }
 tempfile = { workspace = true }
 temp-env = { workspace = true }
-serde_json = { workspace = true }

crates/obs/examples/server.rs

@@ -13,9 +13,9 @@
 // limitations under the License.
 
 use opentelemetry::global;
-use rustfs_obs::{SystemObserver, init_obs};
+use rustfs_obs::init_obs;
 use std::time::{Duration, SystemTime};
-use tracing::{Level, error, info, instrument};
+use tracing::{Level, info, instrument};
 
 #[tokio::main]
 async fn main() {
@@ -43,11 +43,6 @@ async fn run(service_name: String) {
         &[opentelemetry::KeyValue::new("operation", "run")],
     );
 
-    match SystemObserver::init_process_observer().await {
-        Ok(_) => info!("Process observer initialized successfully"),
-        Err(e) => error!("Failed to initialize process observer: {:?}", e),
-    }
-
     put_object("bucket".to_string(), "object".to_string(), "user".to_string()).await;
     info!("Logging is completed");
     tokio::time::sleep(Duration::from_secs(2)).await;

crates/obs/src/global.rs

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{AppConfig, GlobalError, OtelConfig, OtelGuard, SystemObserver, telemetry::init_telemetry};
+use crate::{AppConfig, GlobalError, OtelConfig, OtelGuard, telemetry::init_telemetry};
 use std::sync::{Arc, Mutex};
 use tokio::sync::OnceCell;
-use tracing::{error, info, warn};
+use tracing::{info, warn};
 
 /// Global guard for OpenTelemetry tracing
 static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
@@ -116,17 +116,8 @@ pub async fn init_obs(endpoint: Option<String>) -> Result<OtelGuard, GlobalError
 /// ```
 pub async fn init_obs_with_config(config: &OtelConfig) -> Result<OtelGuard, GlobalError> {
     let otel_guard = init_telemetry(config)?;
-    tokio::spawn(async move {
-        let obs_result = SystemObserver::init_process_observer().await;
-        match obs_result {
-            Ok(_) => {
-                info!(target: "rustfs::obs::system::metrics", "Process observer initialized successfully");
-            }
-            Err(e) => {
-                error!(target: "rustfs::obs::system::metrics", "Failed to initialize process observer: {}", e);
-            }
-        }
-    });
+    // Note: System monitoring has been migrated to rustfs-metrics
+    // Use rustfs_metrics::init_metrics_collectors() for system metrics
     Ok(otel_guard)
 }
 

crates/obs/src/lib.rs

@@ -16,20 +16,12 @@
 //!
 //! provides tools for system and service monitoring
 //!
-//! ## feature mark
-//! - `default`: default monitoring function
-//! - `gpu`: gpu monitoring function
-//! - `full`: includes all functions
+//! ## Features
 //!
-//! to enable gpu monitoring add in cargo toml
-//!
-//! ```toml
-//! # using gpu monitoring
-//! rustfs-obs = { version = "0.1.0", features = ["gpu"] }
-//!
-//! # use all functions
-//! rustfs-obs = { version = "0.1.0", features = ["full"] }
-//! ```
+//! This crate provides observability tools for RustFS:
+//! - Logging with tracing
+//! - Metrics collection
+//! - Distributed tracing
 //!
 //! ## Usage
 //!
@@ -53,16 +45,27 @@
 //! #   // Guard will be dropped here, flushing telemetry data
 //! # }
 //! ```
+//!
+//! ## System Monitoring Migration
+//!
+//! The system monitoring functionality has been migrated to `rustfs-metrics`.
+//! Use `rustfs_metrics::init_metrics_collectors()` for system metrics collection.
+//!
+//! ```ignore
+//! use tokio_util::sync::CancellationToken;
+//! use rustfs_metrics::init_metrics_collectors;
+//!
+//! let token = CancellationToken::new();
+//! init_metrics_collectors(token.clone());
+//! ```
 mod cleaner;
 mod config;
 mod error;
 mod global;
-mod system;
 mod telemetry;
 
 pub use cleaner::*;
 pub use config::*;
 pub use error::*;
 pub use global::*;
-pub use system::SystemObserver;
 pub use telemetry::{OtelGuard, Recorder};

crates/obs/src/system/attributes.rs

@@ -1,58 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::GlobalError;
-use opentelemetry::KeyValue;
-use sysinfo::{Pid, System};
-
-pub(crate) const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid");
-pub(crate) const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name");
-pub(crate) const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path");
-pub(crate) const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command");
-
-/// Struct to hold process attributes
-pub struct ProcessAttributes {
-    pub attributes: Vec<KeyValue>,
-}
-
-impl ProcessAttributes {
-    /// Creates a new instance of `ProcessAttributes` for the given PID.
-    pub fn new(pid: Pid, system: &mut System) -> Result<Self, GlobalError> {
-        system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
-        let process = system
-            .process(pid)
-            .ok_or_else(|| GlobalError::ProcessNotFound(pid.as_u32()))?;
-
-        let attributes = vec![
-            KeyValue::new(PROCESS_PID, pid.as_u32() as i64),
-            KeyValue::new(PROCESS_EXECUTABLE_NAME, process.name().to_os_string().into_string().unwrap_or_default()),
-            KeyValue::new(
-                PROCESS_EXECUTABLE_PATH,
-                process
-                    .exe()
-                    .map(|path| path.to_string_lossy().into_owned())
-                    .unwrap_or_default(),
-            ),
-            KeyValue::new(
-                PROCESS_COMMAND,
-                process
-                    .cmd()
-                    .iter()
-                    .fold(String::new(), |t1, t2| t1 + " " + t2.to_str().unwrap_or_default()),
-            ),
-        ];
-
-        Ok(ProcessAttributes { attributes })
-    }
-}

crates/obs/src/system/collector.rs

@@ -1,173 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::GlobalError;
-use crate::system::attributes::ProcessAttributes;
-use crate::system::metrics::{DIRECTION, INTERFACE, Metrics, STATUS};
-use opentelemetry::KeyValue;
-use std::time::SystemTime;
-use sysinfo::{Networks, Pid, ProcessStatus, System};
-use tokio::time::{Duration, sleep};
-
-/// Collector is responsible for collecting system metrics and attributes.
-/// It uses the sysinfo crate to gather information about the system and processes.
-/// It also uses OpenTelemetry to record metrics.
-pub struct Collector {
-    metrics: Metrics,
-    attributes: ProcessAttributes,
-    #[cfg(feature = "gpu")]
-    gpu_collector: crate::system::gpu::GpuCollector,
-    pid: Pid,
-    system: System,
-    networks: Networks,
-    core_count: usize,
-    interval_ms: u64,
-}
-
-impl Collector {
-    pub fn new(pid: Pid, meter: opentelemetry::metrics::Meter, interval_ms: u64) -> Result<Self, GlobalError> {
-        let mut system = System::new();
-        let attributes = ProcessAttributes::new(pid, &mut system)?;
-        let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?;
-        let metrics = Metrics::new(&meter);
-        #[cfg(feature = "gpu")]
-        let gpu_collector = crate::system::gpu::GpuCollector::new(pid)?;
-        let networks = Networks::new_with_refreshed_list();
-
-        Ok(Collector {
-            metrics,
-            attributes,
-            #[cfg(feature = "gpu")]
-            gpu_collector,
-            pid,
-            system,
-            networks,
-            core_count,
-            interval_ms,
-        })
-    }
-
-    pub async fn run(&mut self) -> Result<(), GlobalError> {
-        loop {
-            self.collect()?;
-            tracing::debug!("Collected metrics for PID: {} ,time: {:?}", self.pid, SystemTime::now());
-            sleep(Duration::from_millis(self.interval_ms)).await;
-        }
-    }
-
-    fn collect(&mut self) -> Result<(), GlobalError> {
-        self.system
-            .refresh_processes(sysinfo::ProcessesToUpdate::Some(&[self.pid]), false);
-
-        // refresh the network interface list and statistics
-        self.networks.refresh(false);
-
-        let process = self
-            .system
-            .process(self.pid)
-            .ok_or_else(|| GlobalError::ProcessNotFound(self.pid.as_u32()))?;
-
-        // CPU metrics
-        let cpu_usage = process.cpu_usage();
-        self.metrics.cpu_usage.record(cpu_usage as f64, &[]);
-        self.metrics
-            .cpu_utilization
-            .record((cpu_usage / self.core_count as f32) as f64, &self.attributes.attributes);
-
-        // Memory metrics
-        self.metrics
-            .memory_usage
-            .record(process.memory() as i64, &self.attributes.attributes);
-        self.metrics
-            .memory_virtual
-            .record(process.virtual_memory() as i64, &self.attributes.attributes);
-
-        // Disk I/O metrics
-        let disk_io = process.disk_usage();
-        self.metrics.disk_io.record(
-            disk_io.read_bytes as i64,
-            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "read")]].concat(),
-        );
-        self.metrics.disk_io.record(
-            disk_io.written_bytes as i64,
-            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "write")]].concat(),
-        );
-
-        // Network I/O indicators (corresponding to /system/network/internode)
-        let mut total_received: i64 = 0;
-        let mut total_transmitted: i64 = 0;
-
-        // statistics by interface
-        for (interface_name, data) in self.networks.iter() {
-            total_received += data.total_received() as i64;
-            total_transmitted += data.total_transmitted() as i64;
-
-            let received = data.received() as i64;
-            let transmitted = data.transmitted() as i64;
-            self.metrics.network_io_per_interface.record(
-                received,
-                &[
-                    &self.attributes.attributes[..],
-                    &[
-                        KeyValue::new(INTERFACE, interface_name.to_string()),
-                        KeyValue::new(DIRECTION, "received"),
-                    ],
-                ]
-                .concat(),
-            );
-            self.metrics.network_io_per_interface.record(
-                transmitted,
-                &[
-                    &self.attributes.attributes[..],
-                    &[
-                        KeyValue::new(INTERFACE, interface_name.to_string()),
-                        KeyValue::new(DIRECTION, "transmitted"),
-                    ],
-                ]
-                .concat(),
-            );
-        }
-        // global statistics
-        self.metrics.network_io.record(
-            total_received,
-            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "received")]].concat(),
-        );
-        self.metrics.network_io.record(
-            total_transmitted,
-            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "transmitted")]].concat(),
-        );
-
-        // Process status indicator (corresponding to /system/process)
-        let status_value = match process.status() {
-            ProcessStatus::Run => 0,
-            ProcessStatus::Sleep => 1,
-            ProcessStatus::Zombie => 2,
-            _ => 3, // other status
-        };
-        self.metrics.process_status.record(
-            status_value,
-            &[
-                &self.attributes.attributes[..],
-                &[KeyValue::new(STATUS, format!("{:?}", process.status()))],
-            ]
-            .concat(),
-        );
-
-        // GPU Metrics (Optional) Non-MacOS
-        #[cfg(feature = "gpu")]
-        self.gpu_collector.collect(&self.metrics, &self.attributes)?;
-
-        Ok(())
-    }
-}

crates/obs/src/system/gpu.rs

@@ -1,57 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::GlobalError;
-use crate::system::attributes::ProcessAttributes;
-use crate::system::metrics::Metrics;
-use nvml_wrapper::Nvml;
-use nvml_wrapper::enums::device::UsedGpuMemory;
-use sysinfo::Pid;
-use tracing::warn;
-
-/// `GpuCollector` is responsible for collecting GPU memory usage metrics.
-pub struct GpuCollector {
-    nvml: Nvml,
-    pid: Pid,
-}
-
-impl GpuCollector {
-    pub fn new(pid: Pid) -> Result<Self, GlobalError> {
-        let nvml = Nvml::init().map_err(|e| GlobalError::GpuInitError(e.to_string()))?;
-        Ok(GpuCollector { nvml, pid })
-    }
-
-    pub fn collect(&self, metrics: &Metrics, attributes: &ProcessAttributes) -> Result<(), GlobalError> {
-        if let Ok(device) = self.nvml.device_by_index(0) {
-            if let Ok(gpu_stats) = device.running_compute_processes() {
-                for stat in gpu_stats.iter() {
-                    if stat.pid == self.pid.as_u32() {
-                        let memory_used = match stat.used_gpu_memory {
-                            UsedGpuMemory::Used(bytes) => bytes,
-                            UsedGpuMemory::Unavailable => 0,
-                        };
-                        metrics.gpu_memory_usage.record(memory_used, &attributes.attributes);
-                        return Ok(());
-                    }
-                }
-            } else {
-                warn!("Could not get GPU stats, recording 0 for GPU memory usage");
-            }
-        } else {
-            return Err(GlobalError::GpuDeviceError("No GPU device found".to_string()));
-        }
-        metrics.gpu_memory_usage.record(0, &attributes.attributes);
-        Ok(())
-    }
-}

crates/obs/src/system/metrics.rs

@@ -1,116 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use opentelemetry::metrics::{Gauge, Meter};
-
-pub(crate) const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
-pub(crate) const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
-pub(crate) const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
-pub(crate) const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
-pub(crate) const PROCESS_DISK_IO: &str = "process.disk.io";
-pub(crate) const PROCESS_NETWORK_IO: &str = "process.network.io";
-pub(crate) const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface";
-pub(crate) const PROCESS_STATUS: &str = "process.status";
-#[cfg(feature = "gpu")]
-pub const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage";
-pub(crate) const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction");
-pub(crate) const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status");
-pub(crate) const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface");
-
-/// `Metrics` struct holds the OpenTelemetry metrics for process monitoring.
-/// It contains various metrics such as CPU usage, memory usage,
-/// disk I/O, network I/O, and process status.
-///
-/// The `Metrics` struct is designed to be used with OpenTelemetry's
-/// metrics API to record and export these metrics.
-///
-/// The `new` method initializes the metrics using the provided
-/// `opentelemetry::metrics::Meter`.
-pub struct Metrics {
-    pub cpu_usage: Gauge<f64>,
-    pub cpu_utilization: Gauge<f64>,
-    pub memory_usage: Gauge<i64>,
-    pub memory_virtual: Gauge<i64>,
-    pub disk_io: Gauge<i64>,
-    pub network_io: Gauge<i64>,
-    pub network_io_per_interface: Gauge<i64>,
-    pub process_status: Gauge<i64>,
-    #[cfg(feature = "gpu")]
-    pub gpu_memory_usage: Gauge<u64>,
-}
-
-impl Metrics {
-    pub fn new(meter: &Meter) -> Self {
-        let cpu_usage = meter
-            .f64_gauge(PROCESS_CPU_USAGE)
-            .with_description("The percentage of CPU in use.")
-            .with_unit("percent")
-            .build();
-        let cpu_utilization = meter
-            .f64_gauge(PROCESS_CPU_UTILIZATION)
-            .with_description("The amount of CPU in use.")
-            .with_unit("percent")
-            .build();
-        let memory_usage = meter
-            .i64_gauge(PROCESS_MEMORY_USAGE)
-            .with_description("The amount of physical memory in use.")
-            .with_unit("byte")
-            .build();
-        let memory_virtual = meter
-            .i64_gauge(PROCESS_MEMORY_VIRTUAL)
-            .with_description("The amount of committed virtual memory.")
-            .with_unit("byte")
-            .build();
-        let disk_io = meter
-            .i64_gauge(PROCESS_DISK_IO)
-            .with_description("Disk bytes transferred.")
-            .with_unit("byte")
-            .build();
-        let network_io = meter
-            .i64_gauge(PROCESS_NETWORK_IO)
-            .with_description("Network bytes transferred.")
-            .with_unit("byte")
-            .build();
-        let network_io_per_interface = meter
-            .i64_gauge(PROCESS_NETWORK_IO_PER_INTERFACE)
-            .with_description("Network bytes transferred (per interface).")
-            .with_unit("byte")
-            .build();
-
-        let process_status = meter
-            .i64_gauge(PROCESS_STATUS)
-            .with_description("Process status (0: Running, 1: Sleeping, 2: Zombie, etc.)")
-            .build();
-
-        #[cfg(feature = "gpu")]
-        let gpu_memory_usage = meter
-            .u64_gauge(PROCESS_GPU_MEMORY_USAGE)
-            .with_description("The amount of physical GPU memory in use.")
-            .with_unit("byte")
-            .build();
-
-        Metrics {
-            cpu_usage,
-            cpu_utilization,
-            memory_usage,
-            memory_virtual,
-            disk_io,
-            network_io,
-            network_io_per_interface,
-            process_status,
-            #[cfg(feature = "gpu")]
-            gpu_memory_usage,
-        }
-    }
-}