feat(admin): add persisted OIDC config APIs (#2267)

Co-authored-by: heihutu <heihutu@gmail.com>

.docker/observability/docker-compose.yml

@@ -15,10 +15,11 @@
 services:
 
   # --- Tracing ---
-
   tempo:
-    image: grafana/tempo:latest
+    image: grafana/tempo:2.10.3
     container_name: tempo
+    depends_on:
+      - redpanda
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
@@ -37,6 +38,38 @@ services:
       timeout: 5s
       retries: 3
       start_period: 15s
+  redpanda:
+    image: redpandadata/redpanda:latest
+    ports:
+      - "9092:9092" # Kafka API for clients
+    command: >
+      redpanda start --overprovisioned
+      --mode=dev-container
+      --kafka-addr=PLAINTEXT://0.0.0.0:9092
+      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
+
+  redpanda-console:
+    image: docker.redpanda.com/redpandadata/console:latest
+    environment:
+      - CONFIG_FILEPATH=/etc/redpanda/redpanda-console-config.yaml
+    volumes:
+      - ./redpanda-console.yaml:/etc/redpanda/redpanda-console-config.yaml
+    ports:
+      - "8080:8080"
+    depends_on:
+      - redpanda
+
+  vulture:
+    image: grafana/tempo-vulture:latest
+    restart: always
+    command:
+      [
+        "-prometheus-listen-address=:8080",
+        "-tempo-query-url=http://tempo:3200",
+        "-tempo-push-url=http://tempo:4317",
+      ]
+    depends_on:
+      - tempo
 
   jaeger:
     image: jaegertracing/jaeger:latest

.docker/observability/grafana/dashboards/rustfs.json

@@ -91,7 +91,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_process_uptime_seconds{job=~\"$job\"}",
+          "expr": "gauge_rustfs_process_uptime_seconds{job=~\"$job\"}",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -223,7 +223,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_buckets_total{job=~\"$job\"})",
+          "expr": "sum(gauge_rustfs_cluster_buckets_total{job=~\"$job\"})",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -289,7 +289,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_objects_total{job=~\"$job\"})",
+          "expr": "sum(gauge_rustfs_cluster_objects_total{job=~\"$job\"})",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -427,7 +427,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"})",
+          "expr": "sum(gauge_rustfs_cluster_capacity_used_bytes{job=~\"$job\"})",
           "legendFormat": "Used",
           "range": true,
           "refId": "A"
@@ -438,7 +438,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
+          "expr": "sum(gauge_rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -450,7 +450,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
+          "expr": "sum(gauge_rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(gauge_rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
           "hide": false,
           "instant": false,
           "legendFormat": "Percent",
@@ -1971,7 +1971,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (drive) (rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"})",
+          "expr": "sum by (drive) (gauge_rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"})",
           "legendFormat": "{{drive}} (bytes)",
           "range": true,
           "refId": "A"
@@ -1982,7 +1982,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (drive) (rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(rustfs_node_disk_total_bytes{job=~\"$job\", drive=~\"$drive\"})",
+          "expr": "sum by (drive) (gauge_rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(gauge_rustfs_node_disk_total_bytes{job=~\"$job\", drive=~\"$drive\"})",
           "hide": false,
           "instant": false,
           "legendFormat": "{{drive}} (percent)",
@@ -2473,7 +2473,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (process_cpu_usage_percent{job=~\"$job\"})",
+          "expr": "sum by (job) (gauge_rustfs_process_cpu_usage{job=~\"$job\"})",
           "legendFormat": "{{job}}",
           "range": true,
           "refId": "A"
@@ -2572,7 +2572,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rustfs_process_memory_bytes{job=~\"$job\"})",
+          "expr": "sum by (job) (gauge_rustfs_process_resident_memory_bytes{job=~\"$job\"})",
           "legendFormat": "{{job}}",
           "range": true,
           "refId": "B"
@@ -2670,7 +2670,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(process_network_io_byte{job=~\"$job\", direction=\"received\"}[5m]))",
+          "expr": "sum by (job) (rate(gauge_rustfs_process_network_io{job=~\"$job\", direction=\"received\"}[5m]))",
           "legendFormat": "RX - {{job}}",
           "range": true,
           "refId": "C"
@@ -2681,7 +2681,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(process_network_io_byte{job=~\"$job\", direction=\"transmitted\"}[5m]))",
+          "expr": "sum by (job) (rate(gauge_rustfs_process_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))",
           "legendFormat": "TX - {{job}}",
           "range": true,
           "refId": "D"
@@ -3339,7 +3339,7 @@
                   "index": 0,
                   "text": "INACTIVE"
                 },
-                "to": 1e-9
+                "to": 1e-09
               },
               "type": "range"
             }
@@ -4048,5 +4048,5 @@
   "timezone": "browser",
   "title": "RustFS",
   "uid": "rustfs-s3",
-  "version": 11
-}
+  "version": 12
+}

.docker/observability/prometheus.yml

@@ -50,6 +50,11 @@ scrape_configs:
     static_configs:
       - targets: [ 'localhost:9090' ]
 
+  - job_name: 'vulture'
+    static_configs:
+      - targets:
+          - 'vulture:8080'
+
 otlp:
   promote_resource_attributes:
     - service.instance.id

.docker/observability/redpanda-console.yaml

@@ -0,0 +1,3 @@
+kafka:
+  brokers:
+    - redpanda:9092

.docker/observability/tempo.yaml

@@ -25,32 +25,18 @@ memberlist:
   join_members:
     - tempo:7946
 
-# Distributor configuration - receives traces and writes directly to ingesters
 distributor:
-  ingester_write_path_enabled: true
-  kafka_write_path_enabled: false
   receivers:
     otlp:
       protocols:
         grpc:
-          endpoint: "tempo:4317"
+          endpoint: "0.0.0.0:4317"
         http:
-          endpoint: "tempo:4318"
-  ring:
-    kvstore:
-      store: memberlist
-
-# Ingester configuration - consumes from Kafka and stores traces
-ingester:
-  lifecycler:
-    ring:
-      kvstore:
-        store: memberlist
-      replication_factor: 1
-    tokens_file_path: /var/tempo/tokens.json
-  trace_idle_period: 10s
-  max_block_bytes: 1_000_000
-  max_block_duration: 5m
+          endpoint: "0.0.0.0:4318"
+  #log_received_spans:
+  #    enabled: true
+  #  log_discarded_spans:
+  #    enabled: true
 
 backend_scheduler:
   provider:
@@ -67,8 +53,7 @@ backend_worker:
       store: memberlist
 
 querier:
-  frontend_worker:
-    frontend_address: tempo:3200
+  query_live_store: true
 
 metrics_generator:
   registry:
@@ -90,9 +75,9 @@ storage:
   trace:
     backend: local
     wal:
-      path: /var/tempo/wal
+      path: /var/tempo/wal # where to store the wal locally
     local:
-      path: /var/tempo/blocks
+      path: /var/tempo/blocks # where to store the traces locally
 
 overrides:
   defaults:
@@ -101,24 +86,14 @@ overrides:
       generate_native_histograms: both
 
 ingest:
-  enabled: false
-  # Disabled because using direct ingester write path
-  # If you want Kafka path, enable this and set:
-  # kafka:
-  #   brokers: [redpanda:9092]
-  #   topic: tempo-ingest
-  #   encoding: protobuf
-  #   consumer_group: tempo-ingest-consumer
+  enabled: true
+  kafka:
+    address: redpanda:9092
+    topic: tempo-ingest
 
 block_builder:
   consume_cycle_duration: 30s
 
-compactor:
-  compaction:
-    block_retention: 168h  # 7 days
-  ring:
-    kvstore:
-      store: memberlist
-
 usage_report:
   reporting_enabled: false
+

.github/dependabot.yml

@@ -26,11 +26,19 @@ updates:
       day: "monday"
       timezone: "Asia/Shanghai"
       time: "08:00"
+    assignees:
+      - "heihutu"
+    reviewers:
+      - "houseme"
+      - "overtrue"
+      - "majinghe"
     ignore:
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"
         versions: [ "0.23.x" ]
+      - dependency-name: "ratelimit"
+        versions: [ "1.x" ]
     groups:
       s3s:
         update-types:

.gitignore

@@ -48,4 +48,5 @@ result*
 *.gz
 rustfs-webdav.code-workspace
 
-.aiexclude
+.aiexclude
+*.bak

Cargo.toml

@@ -148,16 +148,14 @@ rmcp = { version = "1.2.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_bytes = "0.11"
 serde_json = { version = "1.0.149", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
 schemars = "1.2.1"
 
 # Cryptography and Security
 aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
-argon2 = { version = "0.6.0-rc.7" }
+argon2 = { version = "0.6.0-rc.8" }
 blake2 = "0.11.0-rc.5"
-blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
 chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
 hmac = { version = "0.13.0-rc.5" }
@@ -180,15 +178,15 @@ time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros
 
 # Utilities and Tools
 anyhow = "1.0.102"
-arc-swap = "1.8.2"
+arc-swap = "1.9.0"
 astral-tokio-tar = "0.6.0"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.8.15" }
 aws-credential-types = { version = "1.2.14" }
-aws-sdk-s3 = { version = "1.126.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.127.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
-aws-smithy-types = { version = "1.4.6" }
+aws-smithy-types = { version = "1.4.7" }
 backtrace = "0.3.76"
 base64 = "0.22.1"
 base64-simd = "0.8.0"
@@ -202,7 +200,7 @@ crossbeam-queue = "0.3.12"
 crossbeam-channel = "0.5.15"
 crossbeam-deque = "0.8.6"
 crossbeam-utils = "0.8.21"
-datafusion = "52.3.0"
+datafusion = "52.4.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
@@ -224,7 +222,7 @@ matchit = "0.9.1"
 md-5 = "0.11.0-rc.5"
 md5 = "0.8.0"
 mime_guess = "2.0.5"
-moka = { version = "0.12.14", features = ["future"] }
+moka = { version = "0.12.15", features = ["future"] }
 netif = "0.1.6"
 num_cpus = { version = "1.17.0" }
 nvml-wrapper = "0.12.0"
@@ -236,7 +234,7 @@ percent-encoding = "2.3.2"
 pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
 rand = { version = "0.10.0", features = ["serde"] }
-ratelimit = "0.10.0"
+ratelimit = "0.10.1"
 rayon = "1.11.0"
 reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
 reed-solomon-simd = "3.1.0"
@@ -269,19 +267,19 @@ transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
 uuid = { version = "1.22.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
-vaultrs = { version = "0.7.4" }
+vaultrs = { version = "0.8.0" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "8.2.0"
+zip = "8.4.0"
 zstd = "0.13.3"
 
 # Observability and Metrics
 metrics = "0.24.3"
 opentelemetry = { version = "0.31.0" }
 opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
-opentelemetry-otlp = { version = "0.31.0", features = ["gzip-http", "reqwest-rustls"] }
+opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] }
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }

Dockerfile

@@ -72,7 +72,8 @@ LABEL name="RustFS" \
       url="https://rustfs.com" \
       license="Apache-2.0"
 
-RUN apk add --no-cache ca-certificates coreutils curl
+RUN apk update && \
+    apk add --no-cache ca-certificates coreutils curl "zlib>=1.3.2-r0"
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs

crates/appauth/Cargo.toml

@@ -33,3 +33,6 @@ rand.workspace = true
 
 [lints]
 workspace = true
+
+[lib]
+doctest = false

crates/audit/Cargo.toml

@@ -47,3 +47,7 @@ rumqttc = { workspace = true }
 
 [lints]
 workspace = true
+
+[lib]
+test = false
+doctest = false

crates/audit/src/registry.rs

@@ -313,6 +313,12 @@ impl AuditRegistry {
                 }
             }
 
+            if &new_config == config {
+                info!("Audit target configuration unchanged, skip persisting server config");
+                info!(count = successful_targets.len(), "All target processing completed");
+                return Ok(successful_targets);
+            }
+
             let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
                 return Err(AuditError::StorageNotAvailable(
                     "Failed to save target configuration: server storage not initialized".to_string(),

crates/audit/tests/performance_test.rs

@@ -159,7 +159,8 @@ async fn test_audit_log_dispatch_performance() {
 
     let start = Instant::now();
 
-    // Dispatch audit log (should be fast since no targets are configured)
+    // Dispatch audit log against an unstarted system state. Empty config keeps
+    // the audit system stopped, so dispatch should fail fast without targets.
     let result = system.dispatch(Arc::new(audit_entry)).await;
     let elapsed = start.elapsed();
 
@@ -168,8 +169,10 @@ async fn test_audit_log_dispatch_performance() {
     // Should be very fast (sub-millisecond for no targets)
     assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {elapsed:?}");
 
-    // Should succeed even with no targets
-    assert!(result.is_ok(), "Dispatch should succeed with no targets");
+    assert!(
+        matches!(result, Err(AuditError::NotInitialized(_))),
+        "Dispatch on a stopped system should return NotInitialized, got: {result:?}"
+    );
 
     // Clean up
     let _ = system.close().await;
@@ -186,11 +189,11 @@ async fn test_system_state_transitions() {
     let config = rustfs_ecstore::config::Config(std::collections::HashMap::new());
     let start_result = system.start(config).await;
 
-    // Should be running (or failed due to server storage)
+    // Empty config keeps the audit system stopped even when start() succeeds.
     let state = system.get_state().await;
     match start_result {
         Ok(_) => {
-            assert_eq!(state, rustfs_audit::system::AuditSystemState::Running);
+            assert_eq!(state, rustfs_audit::system::AuditSystemState::Stopped);
         }
         Err(_) => {
             // Expected in test environment due to server storage not being initialized

crates/audit/tests/system_integration_test.rs

@@ -29,27 +29,21 @@ async fn test_complete_audit_system_lifecycle() {
     assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
     assert!(!system.is_running().await);
 
-    // 2. Start with empty config (will fail due to no server storage in test)
+    // 2. Start with empty config. The current implementation returns Ok(())
+    // but keeps the system stopped when no audit targets are enabled.
     let config = Config(HashMap::new());
     let start_result = system.start(config).await;
 
-    // Should fail in test environment but state handling should work
+    // State handling should remain consistent for both empty-config success and
+    // storage-unavailable failure paths.
     match start_result {
         Err(AuditError::StorageNotAvailable(_)) => {
             // Expected in test environment
             assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
         }
         Ok(_) => {
-            // If it somehow succeeds, verify running state
-            assert_eq!(system.get_state().await, system::AuditSystemState::Running);
-            assert!(system.is_running().await);
-
-            // Test pause/resume
-            system.pause().await.expect("Should pause successfully");
-            assert_eq!(system.get_state().await, system::AuditSystemState::Paused);
-
-            system.resume().await.expect("Should resume successfully");
-            assert_eq!(system.get_state().await, system::AuditSystemState::Running);
+            assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
+            assert!(!system.is_running().await);
         }
         Err(e) => {
             panic!("Unexpected error: {e}");

crates/checksums/Cargo.toml

@@ -36,3 +36,6 @@ sha2 = { workspace = true }
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }
+
+[lib]
+doctest = false

crates/common/Cargo.toml

@@ -41,3 +41,6 @@ rmp-serde = { workspace = true }
 async-trait = { workspace = true }
 s3s = { workspace = true }
 tracing = { workspace = true }
+
+[lib]
+doctest = false

crates/common/src/internode_metrics.rs

@@ -0,0 +1,170 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use metrics::{counter, gauge};
+use std::sync::{
+    Arc, LazyLock,
+    atomic::{AtomicU64, Ordering},
+};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct InternodeMetricsSnapshot {
+    pub sent_bytes_total: u64,
+    pub recv_bytes_total: u64,
+    pub outgoing_requests_total: u64,
+    pub incoming_requests_total: u64,
+    pub errors_total: u64,
+    pub dial_errors_total: u64,
+    pub dial_avg_time_nanos: u64,
+    pub last_dial_unix_millis: u64,
+}
+
+#[derive(Debug, Default)]
+pub struct InternodeMetrics {
+    sent_bytes_total: AtomicU64,
+    recv_bytes_total: AtomicU64,
+    outgoing_requests_total: AtomicU64,
+    incoming_requests_total: AtomicU64,
+    errors_total: AtomicU64,
+    dial_errors_total: AtomicU64,
+    dial_total_time_nanos: AtomicU64,
+    dial_samples_total: AtomicU64,
+    last_dial_unix_millis: AtomicU64,
+}
+
+impl InternodeMetrics {
+    pub fn record_sent_bytes(&self, bytes: usize) {
+        let bytes = bytes as u64;
+        if bytes == 0 {
+            return;
+        }
+        self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        counter!("rustfs.internode.sent.bytes.total").increment(bytes);
+    }
+
+    pub fn record_recv_bytes(&self, bytes: usize) {
+        let bytes = bytes as u64;
+        if bytes == 0 {
+            return;
+        }
+        self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        counter!("rustfs.internode.recv.bytes.total").increment(bytes);
+    }
+
+    pub fn record_outgoing_request(&self) {
+        self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.requests.outgoing.total").increment(1);
+    }
+
+    pub fn record_incoming_request(&self) {
+        self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.requests.incoming.total").increment(1);
+    }
+
+    pub fn record_error(&self) {
+        self.errors_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.errors.total").increment(1);
+    }
+
+    pub fn record_dial_result(&self, duration: Duration, success: bool) {
+        let elapsed_nanos = duration.as_nanos().min(u128::from(u64::MAX)) as u64;
+        self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed);
+        let samples = self.dial_samples_total.fetch_add(1, Ordering::Relaxed) + 1;
+        let total = self.dial_total_time_nanos.load(Ordering::Relaxed);
+        gauge!("rustfs.internode.dial.avg_time.nanos").set(total as f64 / samples as f64);
+
+        if !success {
+            self.dial_errors_total.fetch_add(1, Ordering::Relaxed);
+            counter!("rustfs.internode.dial.errors.total").increment(1);
+        }
+
+        let now_ms = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_millis()
+            .min(u128::from(u64::MAX)) as u64;
+        self.last_dial_unix_millis.store(now_ms, Ordering::Relaxed);
+    }
+
+    pub fn snapshot(&self) -> InternodeMetricsSnapshot {
+        let dial_samples_total = self.dial_samples_total.load(Ordering::Relaxed);
+        let dial_total_time_nanos = self.dial_total_time_nanos.load(Ordering::Relaxed);
+        let dial_avg_time_nanos = if dial_samples_total == 0 {
+            0
+        } else {
+            dial_total_time_nanos / dial_samples_total
+        };
+
+        InternodeMetricsSnapshot {
+            sent_bytes_total: self.sent_bytes_total.load(Ordering::Relaxed),
+            recv_bytes_total: self.recv_bytes_total.load(Ordering::Relaxed),
+            outgoing_requests_total: self.outgoing_requests_total.load(Ordering::Relaxed),
+            incoming_requests_total: self.incoming_requests_total.load(Ordering::Relaxed),
+            errors_total: self.errors_total.load(Ordering::Relaxed),
+            dial_errors_total: self.dial_errors_total.load(Ordering::Relaxed),
+            dial_avg_time_nanos,
+            last_dial_unix_millis: self.last_dial_unix_millis.load(Ordering::Relaxed),
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn reset_for_test(&self) {
+        self.sent_bytes_total.store(0, Ordering::Relaxed);
+        self.recv_bytes_total.store(0, Ordering::Relaxed);
+        self.outgoing_requests_total.store(0, Ordering::Relaxed);
+        self.incoming_requests_total.store(0, Ordering::Relaxed);
+        self.errors_total.store(0, Ordering::Relaxed);
+        self.dial_errors_total.store(0, Ordering::Relaxed);
+        self.dial_total_time_nanos.store(0, Ordering::Relaxed);
+        self.dial_samples_total.store(0, Ordering::Relaxed);
+        self.last_dial_unix_millis.store(0, Ordering::Relaxed);
+    }
+}
+
+pub fn global_internode_metrics() -> &'static Arc<InternodeMetrics> {
+    static GLOBAL_INTERNODE_METRICS: LazyLock<Arc<InternodeMetrics>> = LazyLock::new(|| Arc::new(InternodeMetrics::default()));
+    &GLOBAL_INTERNODE_METRICS
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn snapshot_reports_recorded_values() {
+        let metrics = global_internode_metrics();
+        metrics.reset_for_test();
+
+        metrics.record_sent_bytes(64);
+        metrics.record_recv_bytes(32);
+        metrics.record_outgoing_request();
+        metrics.record_incoming_request();
+        metrics.record_error();
+        metrics.record_dial_result(Duration::from_millis(9), true);
+        metrics.record_dial_result(Duration::from_millis(3), false);
+
+        let snapshot = metrics.snapshot();
+        assert_eq!(snapshot.sent_bytes_total, 64);
+        assert_eq!(snapshot.recv_bytes_total, 32);
+        assert_eq!(snapshot.outgoing_requests_total, 1);
+        assert_eq!(snapshot.incoming_requests_total, 1);
+        assert_eq!(snapshot.errors_total, 1);
+        assert_eq!(snapshot.dial_errors_total, 1);
+        assert_eq!(snapshot.dial_avg_time_nanos, 6_000_000);
+        assert!(snapshot.last_dial_unix_millis > 0);
+
+        metrics.reset_for_test();
+    }
+}

crates/common/src/lib.rs

@@ -17,6 +17,7 @@ pub mod bucket_stats;
 pub mod data_usage;
 pub mod globals;
 pub mod heal_channel;
+pub mod internode_metrics;
 pub mod last_minute;
 pub mod metrics;
 mod readiness;

crates/common/src/readiness.rs

@@ -60,6 +60,23 @@ impl GlobalReadiness {
     pub fn is_ready(&self) -> bool {
         self.status.load(Ordering::SeqCst) == SystemStage::FullReady as u8
     }
+
+    /// Get the current system stage
+    /// # Returns
+    /// The current SystemStage of the service
+    pub fn current_stage(&self) -> SystemStage {
+        match self.status.load(Ordering::SeqCst) {
+            0 => SystemStage::Booting,
+            1 => SystemStage::StorageReady,
+            2 => SystemStage::IamReady,
+            3 => SystemStage::FullReady,
+            invalid => {
+                debug_assert!(false, "GlobalReadiness::current_stage: invalid status value {}", invalid);
+                // Fallback to the most conservative stage on invalid values
+                SystemStage::Booting
+            }
+        }
+    }
 }
 
 #[cfg(test)]

crates/config/Cargo.toml

@@ -37,3 +37,6 @@ constants = ["dep:const-str"]
 notify = ["dep:const-str", "constants"]
 observability = ["constants"]
 opa = ["constants"]
+
+[lib]
+doctest = false

crates/config/src/constants/app.rs

@@ -131,9 +131,61 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 /// Environment variable for server volumes.
 pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
 
+/// Environment variable for server access key.
+pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
+
+/// Environment variable for server access key file.
+pub const ENV_RUSTFS_ACCESS_KEY_FILE: &str = "RUSTFS_ACCESS_KEY_FILE";
+
+/// Environment variable for server root user.
+pub const ENV_RUSTFS_ROOT_USER: &str = "RUSTFS_ROOT_USER";
+
+/// Environment variable for server secret key.
+pub const ENV_RUSTFS_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
+
+/// Environment variable for server secret key file.
+pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE";
+
+/// Environment variable for server root password.
+pub const ENV_RUSTFS_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD";
+
+/// Environment variable for server OBS endpoint.
+pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
+
+/// Environment variable for console server enable.
+pub const ENV_RUSTFS_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE";
+
+/// Environment variable for console server address.
+pub const ENV_RUSTFS_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS";
+
 /// Environment variable for server tls path.
 pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH";
 
+/// Environment variable for server KMS enable.
+pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
+
+/// Default KMS enable for server-side encryption
+/// This is the default value for enabling KMS encryption for server-side encryption.
+/// Default value: false
+pub const DEFAULT_KMS_ENABLE: bool = false;
+
+/// Environment variable for server KMS backend.
+pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND";
+
+/// Default KMS backend for server-side encryption
+/// This is the default KMS backend for server-side encryption.
+/// Default value: local
+pub const DEFAULT_KMS_BACKEND: &str = "local";
+
+/// Environment variable for selecting the buffer profile used for adaptive buffer sizing.
+pub const ENV_RUSTFS_BUFFER_PROFILE: &str = "RUSTFS_BUFFER_PROFILE";
+
+/// Default buffer profile for adaptive buffer sizing
+/// This is the default buffer profile for adaptive buffer sizing.
+/// It is used to identify the workload profile for adaptive buffer sizing.
+/// Default value: GeneralPurpose
+pub const DEFAULT_BUFFER_PROFILE: &str = "GeneralPurpose";
+
 /// Default value for the server TLS path if `ENV_RUSTFS_TLS_PATH` is not set.
 pub const DEFAULT_RUSTFS_TLS_PATH: &str = "";
 

crates/config/src/constants/mod.rs

@@ -28,3 +28,4 @@ pub(crate) mod runtime;
 pub(crate) mod scanner;
 pub(crate) mod targets;
 pub(crate) mod tls;
+pub(crate) mod workload;

crates/config/src/constants/object.rs

@@ -180,3 +180,265 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
 
 /// Default: bitrot verification is enabled on GetObject reads (do not skip).
 pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
+
+// =============================================================================
+// Concurrent Request Fix - Timeout and Backpressure Configuration
+// =============================================================================
+
+/// Environment variable for GetObject request timeout in seconds.
+///
+/// When a GetObject request exceeds this duration, it will be cancelled
+/// and return a 504 Gateway Timeout error. This prevents requests from
+/// hanging indefinitely due to deadlocks or resource exhaustion.
+///
+/// Default: 30 seconds (can be overridden by `RUSTFS_OBJECT_GET_TIMEOUT`).
+/// Set to 0 to disable timeout (not recommended for production).
+pub const ENV_OBJECT_GET_TIMEOUT: &str = "RUSTFS_OBJECT_GET_TIMEOUT";
+
+/// Default GetObject request timeout in seconds.
+///
+/// This value balances between allowing large object transfers to complete
+/// and preventing indefinite hangs. For 20-26MB objects with concurrent
+/// range reads, 30 seconds should be sufficient under normal conditions.
+pub const DEFAULT_OBJECT_GET_TIMEOUT: u64 = 30;
+
+/// Environment variable for disk read operation timeout in seconds.
+///
+/// Individual disk read operations that exceed this duration will be
+/// cancelled and treated as failures. This helps detect slow or hung
+/// disks without waiting indefinitely.
+///
+/// Default: 10 seconds (can be overridden by `RUSTFS_OBJECT_DISK_READ_TIMEOUT`).
+pub const ENV_OBJECT_DISK_READ_TIMEOUT: &str = "RUSTFS_OBJECT_DISK_READ_TIMEOUT";
+
+/// Default disk read timeout in seconds.
+pub const DEFAULT_OBJECT_DISK_READ_TIMEOUT: u64 = 10;
+
+/// Environment variable for duplex pipe buffer size in bytes.
+///
+/// The duplex pipe connects the disk read task to the HTTP response stream.
+/// A larger buffer reduces backpressure but increases memory usage.
+/// For large objects (20-26MB), a 4MB buffer provides good throughput.
+///
+/// Default: 4194304 (4 MB, can be overridden by `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE`).
+/// Minimum recommended: 1048576 (1 MB).
+pub const ENV_OBJECT_DUPLEX_BUFFER_SIZE: &str = "RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE";
+
+/// Default duplex buffer size: 4 MB.
+///
+/// This is 4x larger than the original 1 MB buffer, providing better
+/// handling of large objects and reducing backpressure-related hangs.
+pub const DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE: usize = 4 * 1024 * 1024;
+
+/// Environment variable for I/O buffer size in bytes.
+///
+/// This controls the buffer size used for individual I/O operations.
+/// A larger buffer improves throughput for sequential reads but may
+/// increase latency for small random reads.
+///
+/// Default: 131072 (128 KB, can be overridden by `RUSTFS_OBJECT_IO_BUFFER_SIZE`).
+pub const ENV_OBJECT_IO_BUFFER_SIZE: &str = "RUSTFS_OBJECT_IO_BUFFER_SIZE";
+
+/// Default I/O buffer size: 128 KB.
+pub const DEFAULT_OBJECT_IO_BUFFER_SIZE: usize = 128 * 1024;
+
+/// Environment variable to enable/disable lock optimization.
+///
+/// When enabled, read locks are released immediately after metadata
+/// is read, rather than being held for the entire data transfer.
+/// This significantly reduces lock contention under high concurrency.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE`).
+pub const ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE: &str = "RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE";
+
+/// Default: lock optimization is enabled.
+pub const DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE: bool = true;
+
+/// Environment variable to enable/disable priority-based I/O scheduling.
+///
+/// When enabled, smaller requests (< 1MB) are given higher priority
+/// than larger requests (> 10MB), preventing "starvation" of small
+/// requests by large ones.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE`).
+pub const ENV_OBJECT_PRIORITY_SCHEDULING_ENABLE: &str = "RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE";
+
+/// Default: priority scheduling is enabled.
+pub const DEFAULT_OBJECT_PRIORITY_SCHEDULING_ENABLE: bool = true;
+
+/// Environment variable to enable/disable deadlock detection.
+///
+/// When enabled, the system monitors active requests and detects
+/// potential deadlock situations (circular lock wait chains).
+/// This has some performance overhead and is intended for debugging.
+///
+/// Default: false (disabled, can be overridden by `RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE`).
+pub const ENV_OBJECT_DEADLOCK_DETECTION_ENABLE: &str = "RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE";
+
+/// Default: deadlock detection is disabled for performance.
+pub const DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE: bool = false;
+
+/// Environment variable for deadlock detection check interval in seconds.
+///
+/// How often the deadlock detector analyzes the lock wait graph.
+/// More frequent checks detect deadlocks faster but use more CPU.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL`).
+pub const ENV_OBJECT_DEADLOCK_CHECK_INTERVAL: &str = "RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL";
+
+/// Default deadlock check interval: 5 seconds.
+pub const DEFAULT_OBJECT_DEADLOCK_CHECK_INTERVAL: u64 = 5;
+
+/// Environment variable for deadlock detection hang threshold in seconds.
+///
+/// Requests that have been running longer than this threshold are
+/// considered "potentially hung" and included in deadlock analysis.
+///
+/// Default: 10 seconds (can be overridden by `RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD`).
+pub const ENV_OBJECT_DEADLOCK_HANG_THRESHOLD: &str = "RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD";
+
+/// Default hang threshold: 10 seconds.
+pub const DEFAULT_OBJECT_DEADLOCK_HANG_THRESHOLD: u64 = 10;
+
+/// Environment variable for backpressure high watermark percentage.
+///
+/// When buffer usage exceeds this percentage, the system enters
+/// "high watermark" state and may apply backpressure to producers.
+///
+/// Default: 80 (80%, can be overridden by `RUSTFS_OBJECT_BACKPRESSURE_HIGH_WATERMARK`).
+pub const ENV_OBJECT_BACKPRESSURE_HIGH_WATERMARK: &str = "RUSTFS_OBJECT_BACKPRESSURE_HIGH_WATERMARK";
+
+/// Default high watermark: 80%.
+pub const DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK: u32 = 80;
+
+/// Environment variable for backpressure low watermark percentage.
+///
+/// When buffer usage drops below this percentage after being in
+/// high watermark state, backpressure is released.
+///
+/// Default: 50 (50%, can be overridden by `RUSTFS_OBJECT_BACKPRESSURE_LOW_WATERMARK`).
+pub const ENV_OBJECT_BACKPRESSURE_LOW_WATERMARK: &str = "RUSTFS_OBJECT_BACKPRESSURE_LOW_WATERMARK";
+
+/// Default low watermark: 50%.
+pub const DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK: u32 = 50;
+
+/// Environment variable for lock acquisition timeout in seconds.
+///
+/// When a lock cannot be acquired within this duration, the operation
+/// will fail with a timeout error. This prevents indefinite waiting
+/// for locks that may never be released due to deadlocks.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT`).
+pub const ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT: &str = "RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT";
+
+/// Default lock acquisition timeout: 5 seconds.
+pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5;
+
+// ============================================================================
+// I/O priority scheduling configuration
+// ============================================================================
+
+/// Environment variable for I/O high priority size threshold in bytes.
+///
+/// Requests smaller than this threshold are classified as high priority.
+/// High priority requests are processed first to prevent starvation of small requests.
+///
+/// Default: 1048576 (1 MB, can be overridden by `RUSTFS_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD`).
+pub const ENV_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD: &str = "RUSTFS_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD";
+
+/// Default high priority size threshold: 1 MB.
+pub const DEFAULT_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD: usize = 1024 * 1024;
+
+/// Environment variable for I/O low priority size threshold in bytes.
+///
+/// Requests larger than this threshold are classified as low priority.
+/// Low priority requests are processed last to avoid blocking small requests.
+///
+/// Default: 104857600 (100 MB, can be overridden by `RUSTFS_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD`).
+pub const ENV_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD: &str = "RUSTFS_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD";
+
+/// Default low priority size threshold: 100 MB.
+pub const DEFAULT_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD: usize = 100 * 1024 * 1024;
+
+/// Environment variable for high priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the high priority queue.
+///
+/// Default: 32 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_HIGH_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_HIGH_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_HIGH_CAPACITY";
+
+/// Default high priority queue capacity: 32.
+pub const DEFAULT_OBJECT_IO_QUEUE_HIGH_CAPACITY: usize = 32;
+
+/// Environment variable for normal priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the normal priority queue.
+///
+/// Default: 64 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_NORMAL_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_NORMAL_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_NORMAL_CAPACITY";
+
+/// Default normal priority queue capacity: 64.
+pub const DEFAULT_OBJECT_IO_QUEUE_NORMAL_CAPACITY: usize = 64;
+
+/// Environment variable for low priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the low priority queue.
+///
+/// Default: 16 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_LOW_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_LOW_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_LOW_CAPACITY";
+
+/// Default low priority queue capacity: 16.
+pub const DEFAULT_OBJECT_IO_QUEUE_LOW_CAPACITY: usize = 16;
+
+/// Environment variable for starvation prevention check interval in milliseconds.
+///
+/// How often the system checks for starving low-priority requests.
+/// When a low-priority request has been waiting longer than the starvation threshold,
+/// it is promoted to normal priority.
+///
+/// Default: 100 ms (can be overridden by `RUSTFS_OBJECT_IO_STARVATION_PREVENTION_INTERVAL`).
+pub const ENV_OBJECT_IO_STARVATION_PREVENTION_INTERVAL: &str = "RUSTFS_OBJECT_IO_STARVATION_PREVENTION_INTERVAL";
+
+/// Default starvation prevention interval: 100 ms.
+pub const DEFAULT_OBJECT_IO_STARVATION_PREVENTION_INTERVAL: u64 = 100;
+
+/// Environment variable for starvation threshold in seconds.
+///
+/// Maximum time a low-priority request can wait before being promoted to normal priority.
+/// This prevents indefinite starvation of low-priority requests.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_IO_STARVATION_THRESHOLD_SECS`).
+pub const ENV_OBJECT_IO_STARVATION_THRESHOLD_SECS: &str = "RUSTFS_OBJECT_IO_STARVATION_THRESHOLD_SECS";
+
+/// Default starvation threshold: 5 seconds.
+pub const DEFAULT_OBJECT_IO_STARVATION_THRESHOLD_SECS: u64 = 5;
+
+/// Environment variable for load sampling window size.
+///
+/// Number of recent samples used to calculate I/O load metrics.
+///
+/// Default: 100 samples (can be overridden by `RUSTFS_OBJECT_IO_LOAD_SAMPLE_WINDOW`).
+pub const ENV_OBJECT_IO_LOAD_SAMPLE_WINDOW: &str = "RUSTFS_OBJECT_IO_LOAD_SAMPLE_WINDOW";
+
+/// Default load sampling window: 100 samples.
+pub const DEFAULT_OBJECT_IO_LOAD_SAMPLE_WINDOW: usize = 100;
+
+/// Environment variable for high load wait time threshold in milliseconds.
+///
+/// When average wait time exceeds this threshold, the system is considered to be under high load.
+///
+/// Default: 50 ms (can be overridden by `RUSTFS_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS`).
+pub const ENV_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS: &str = "RUSTFS_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS";
+
+/// Default high load threshold: 50 ms.
+pub const DEFAULT_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS: u64 = 50;
+
+/// Environment variable for low load wait time threshold in milliseconds.
+///
+/// When average wait time is below this threshold, the system is considered to be under low load.
+///
+/// Default: 10 ms (can be overridden by `RUSTFS_OBJECT_IO_LOAD_LOW_THRESHOLD_MS`).
+pub const ENV_OBJECT_IO_LOAD_LOW_THRESHOLD_MS: &str = "RUSTFS_OBJECT_IO_LOAD_LOW_THRESHOLD_MS";
+
+/// Default low load threshold: 10 ms.
+pub const DEFAULT_OBJECT_IO_LOAD_LOW_THRESHOLD_MS: u64 = 10;

crates/config/src/constants/workload.rs

@@ -0,0 +1,70 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Workload profile buffer configuration constants.
+//!
+//! This module defines environment variable keys and default values for
+//! custom buffer profile configuration.
+
+use crate::{KI_B, MI_B};
+
+/// Environment variable for minimum buffer size
+/// Default: 64KB (65536 bytes)
+pub const ENV_RUSTFS_BUFFER_MIN_SIZE: &str = "RUSTFS_BUFFER_MIN_SIZE";
+
+/// Environment variable for maximum buffer size
+/// Default: 1MB (1048576 bytes)
+pub const ENV_RUSTFS_BUFFER_MAX_SIZE: &str = "RUSTFS_BUFFER_MAX_SIZE";
+
+/// Environment variable for default buffer size (used when file size is unknown)
+/// Default: 256KB (262144 bytes)
+pub const ENV_RUSTFS_BUFFER_DEFAULT_SIZE: &str = "RUSTFS_BUFFER_DEFAULT_SIZE";
+
+/// Default minimum buffer size: 64KB
+pub const DEFAULT_BUFFER_MIN_SIZE: usize = 64 * KI_B;
+
+/// Default maximum buffer size: 1MB
+pub const DEFAULT_BUFFER_MAX_SIZE: usize = MI_B;
+
+/// Default buffer size for unknown file size: 256KB
+pub const DEFAULT_BUFFER_UNKNOWN_SIZE: usize = 256 * KI_B;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::MI_B;
+
+    #[test]
+    fn test_default_values() {
+        assert_eq!(DEFAULT_BUFFER_MIN_SIZE, 65536); // 64KB
+        assert_eq!(DEFAULT_BUFFER_MAX_SIZE, 1048576); // 1MB
+        assert_eq!(DEFAULT_BUFFER_UNKNOWN_SIZE, 262144); // 256KB
+    }
+
+    #[test]
+    fn test_constants() {
+        assert_eq!(KI_B, 1024);
+        assert_eq!(MI_B, 1024 * 1024);
+        assert_eq!(64 * KI_B, DEFAULT_BUFFER_MIN_SIZE);
+        assert_eq!(MI_B, DEFAULT_BUFFER_MAX_SIZE);
+        assert_eq!(256 * KI_B, DEFAULT_BUFFER_UNKNOWN_SIZE);
+    }
+
+    #[test]
+    fn test_env_var_names() {
+        assert_eq!(ENV_RUSTFS_BUFFER_MIN_SIZE, "RUSTFS_BUFFER_MIN_SIZE");
+        assert_eq!(ENV_RUSTFS_BUFFER_MAX_SIZE, "RUSTFS_BUFFER_MAX_SIZE");
+        assert_eq!(ENV_RUSTFS_BUFFER_DEFAULT_SIZE, "RUSTFS_BUFFER_DEFAULT_SIZE");
+    }
+}

crates/config/src/lib.rs

@@ -45,6 +45,8 @@ pub use constants::targets::*;
 #[cfg(feature = "constants")]
 pub use constants::tls::*;
 #[cfg(feature = "constants")]
+pub use constants::workload::*;
+#[cfg(feature = "constants")]
 pub mod oidc {
     pub use super::constants::oidc::*;
 }

crates/crypto/Cargo.toml

@@ -56,3 +56,6 @@ crypto = [
     "dep:rand",
     "dep:sha2",
 ]
+
+[lib]
+doctest = false

crates/e2e_test/src/delete_objects_versioning_test.rs

@@ -16,15 +16,13 @@
 //! "In a versioned Bucket, DeleteMarkers are not appearing straight after
 //! a delete_objects is called."
 //!
-//! Root cause: `delete_versions_internal` wrote new xl.meta to disk via
-//! `write_all_private` without invalidating the `GlobalFileCache`. Subsequent
-//! calls to `read_metadata` returned the stale cached xl.meta (without the
-//! delete marker), making `list_object_versions` show the old version as
-//! `IsLatest=true` rather than the new delete marker.
+//! Root cause: metadata updates could become temporarily invisible to
+//! `list_object_versions`, so the old version was still reported as
+//! `IsLatest=true` instead of the newly-created delete marker.
 //!
-//! Fix: `write_all_private` now calls `get_global_file_cache().invalidate()`
-//! after every successful write, and `rename_data` also invalidates the cache
-//! for the destination path after the atomic rename.
+//! Fix: metadata write, delete, and rename paths now make the updated
+//! `xl.meta` immediately visible, and the old file-cache shortcut has been
+//! removed from the read path.
 
 #[cfg(test)]
 mod tests {

crates/e2e_test/src/lib.rs

@@ -66,6 +66,11 @@ mod compression_test;
 // Regression test for Issue #1878: DeleteMarkers not visible immediately after delete_objects
 #[cfg(test)]
 mod delete_objects_versioning_test;
+
+// Regression test for Issue #2252: ListObjectVersions misses newest version after put -> delete -> put
+#[cfg(test)]
+mod list_object_versions_regression_test;
+
 #[cfg(test)]
 mod protocols;
 

crates/e2e_test/src/list_object_versions_regression_test.rs

@@ -0,0 +1,182 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for Issue #2252:
+//! "ListObjectVersions misses the newest version after create -> delete -> create."
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use serial_test::serial;
+    use tracing::info;
+
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_immediately_returns_latest_put_after_delete_marker() {
+        init_logging();
+        info!("🧪 TEST: ListObjectVersions returns the newest version immediately after put -> delete -> put");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-object-versions-2252";
+        let key = "test-prefix/test-object.txt";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to enable versioning");
+
+        let first_put = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"first version"))
+            .send()
+            .await
+            .expect("Failed to put first object version");
+        let first_version_id = first_put
+            .version_id()
+            .map(str::to_string)
+            .expect("First put should return a version_id");
+
+        let delete_resp = client
+            .delete_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("Failed to create delete marker");
+        let delete_marker_version_id = delete_resp
+            .version_id()
+            .map(str::to_string)
+            .expect("DeleteObject should return a delete marker version_id");
+
+        let second_put = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"second version"))
+            .send()
+            .await
+            .expect("Failed to put second object version");
+        let second_version_id = second_put
+            .version_id()
+            .map(str::to_string)
+            .expect("Second put should return a version_id");
+
+        let first_listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions immediately after second put");
+        let second_listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions a second time");
+
+        let first_versions = first_listing.versions().to_vec();
+        let first_delete_markers = first_listing.delete_markers().to_vec();
+        let second_versions = second_listing.versions().to_vec();
+        let second_delete_markers = second_listing.delete_markers().to_vec();
+
+        info!(
+            "First listing: {} versions, {} delete markers",
+            first_versions.len(),
+            first_delete_markers.len()
+        );
+        info!(
+            "Second listing: {} versions, {} delete markers",
+            second_versions.len(),
+            second_delete_markers.len()
+        );
+
+        assert_eq!(
+            first_versions.len(),
+            2,
+            "First ListObjectVersions call should return both object versions immediately (regression #2252)"
+        );
+        assert_eq!(
+            first_delete_markers.len(),
+            1,
+            "First ListObjectVersions call should return the delete marker immediately (regression #2252)"
+        );
+        assert_eq!(
+            second_versions.len(),
+            2,
+            "Second ListObjectVersions call should still return both object versions"
+        );
+        assert_eq!(
+            second_delete_markers.len(),
+            1,
+            "Second ListObjectVersions call should still return the delete marker"
+        );
+
+        let first_latest_version = first_versions
+            .iter()
+            .find(|version| version.version_id() == Some(second_version_id.as_str()))
+            .expect("First listing should include the newest object version");
+        assert_eq!(
+            first_latest_version.is_latest(),
+            Some(true),
+            "Newest object version should be latest on the first listing"
+        );
+
+        let first_original_version = first_versions
+            .iter()
+            .find(|version| version.version_id() == Some(first_version_id.as_str()))
+            .expect("First listing should include the original object version");
+        assert_eq!(
+            first_original_version.is_latest(),
+            Some(false),
+            "Original object version should no longer be latest"
+        );
+
+        let first_delete_marker = first_delete_markers
+            .iter()
+            .find(|marker| marker.version_id() == Some(delete_marker_version_id.as_str()))
+            .expect("First listing should include the delete marker");
+        assert_eq!(
+            first_delete_marker.is_latest(),
+            Some(false),
+            "Delete marker should no longer be latest after the second put"
+        );
+    }
+}

crates/e2e_test/src/reliant/grpc_lock_client.rs

@@ -61,6 +61,7 @@ impl GrpcLockClient {
             metadata: LockMetadata::default(),
             priority: LockPriority::Normal,
             deadlock_detection: false,
+            suppress_contention_logs: false,
         }
     }
 }

crates/ecstore/Cargo.toml

@@ -74,7 +74,6 @@ regex = { workspace = true }
 path-absolutize = { workspace = true }
 rmp.workspace = true
 rmp-serde.workspace = true
-serde_bytes.workspace = true
 tokio-util = { workspace = true, features = ["io", "compat"] }
 base64 = { workspace = true }
 hmac = { workspace = true }
@@ -117,6 +116,9 @@ faster-hex = { workspace = true }
 ratelimit = { workspace = true }
 aws-smithy-http-client.workspace = true
 
+# Observability and Metrics
+metrics = { workspace = true }
+
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 criterion = { workspace = true, features = ["html_reports"] }
@@ -134,3 +136,6 @@ harness = false
 [[bench]]
 name = "comparison_benchmark"
 harness = false
+
+[lib]
+doctest = false

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -33,7 +33,9 @@ use crate::global::GLOBAL_LocalNodeName;
 use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
 use crate::store::ECStore;
 use crate::store_api::StorageAPI;
-use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete};
+use crate::store_api::{
+    GetObjectReader, HTTPRangeSpec, ListOperations, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete,
+};
 use crate::tier::warm_backend::WarmBackendGetOpts;
 use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
 use bytes::BytesMut;
@@ -646,18 +648,48 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
 }
 
 pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
-    let lc = GLOBAL_LifecycleSys.get(&oi.bucket).await;
-    if !lc.is_none() {
-        let event = lc.expect("err").eval(&oi.to_lifecycle_opts()).await;
-        match event.action {
-            IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
-                if oi.delete_marker || oi.is_dir {
-                    return;
-                }
-                GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await;
-            }
-            _ => (),
+    if let Some(lc) = GLOBAL_LifecycleSys.get(&oi.bucket).await {
+        enqueue_transition_with_lifecycle(oi, &lc, &src).await;
+    }
+}
+
+pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
+    let Some(lc) = GLOBAL_LifecycleSys.get(bucket).await else {
+        return Ok(());
+    };
+    let mut marker = None;
+    let mut version_marker = None;
+    let src = LcEventSrc::Scanner;
+
+    loop {
+        let page = api
+            .clone()
+            .list_object_versions(bucket, "", marker.clone(), version_marker.clone(), None, 1000)
+            .await?;
+
+        for object in &page.objects {
+            enqueue_transition_with_lifecycle(object, &lc, &src).await;
         }
+
+        if !page.is_truncated {
+            return Ok(());
+        }
+
+        marker = page.next_marker;
+        version_marker = page.next_version_idmarker;
+    }
+}
+
+async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &BucketLifecycleConfiguration, src: &LcEventSrc) {
+    let event = lc.eval(&oi.to_lifecycle_opts()).await;
+    match event.action {
+        IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
+            if oi.delete_marker || oi.is_dir {
+                return;
+            }
+            GLOBAL_TransitionState.queue_transition_task(oi, &event, src).await;
+        }
+        _ => (),
     }
 }
 

crates/ecstore/src/bucket/lifecycle/lifecycle.rs

@@ -768,7 +768,7 @@ impl LifecycleCalculate for NoncurrentVersionTransition {
 #[async_trait::async_trait]
 impl LifecycleCalculate for Transition {
     fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
-        if !obj.is_latest || self.days.is_none() {
+        if !obj.is_latest {
             return None;
         }
 
@@ -1296,6 +1296,45 @@ mod tests {
         assert_eq!(event.storage_class, "COLDTIER44");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn eval_inner_transitions_latest_object_after_date_due() {
+        let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
+        let transition_date = base_time - Duration::days(1);
+        let lc = BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: None,
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: None,
+                filter: None,
+                id: Some("transition-date".to_string()),
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: Some(vec![Transition {
+                    days: None,
+                    date: Some(transition_date.into()),
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]),
+            }],
+        };
+
+        let opts = ObjectOpts {
+            name: "obj".to_string(),
+            mod_time: Some(base_time),
+            is_latest: true,
+            transition_status: "".to_string(),
+            ..Default::default()
+        };
+        let event = lc.eval_inner(&opts, base_time + Duration::days(1), 0).await;
+
+        assert_eq!(event.action, IlmAction::TransitionAction);
+        assert_eq!(event.rule_id, "transition-date");
+        assert_eq!(event.storage_class, "WARM");
+    }
+
     #[tokio::test]
     #[serial]
     async fn eval_inner_expires_noncurrent_version_after_due() {

crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs

@@ -51,6 +51,10 @@ impl LastDayTierStats {
         self.bins[now_idx] = self.bins[now_idx].add(&ts);
     }
 
+    pub fn total(&self) -> TierStats {
+        self.bins.iter().fold(TierStats::default(), |acc, bin| acc.add(bin))
+    }
+
     fn forward_to(&mut self, t: &mut OffsetDateTime) {
         if t.unix_timestamp() == 0 {
             *t = OffsetDateTime::now_utc();
@@ -99,4 +103,30 @@ impl LastDayTierStats {
 }
 
 #[cfg(test)]
-mod test {}
+mod test {
+    use super::*;
+
+    #[test]
+    fn total_sums_all_recorded_stats() {
+        let mut stats = LastDayTierStats::default();
+        stats.add_stats(TierStats {
+            total_size: 10,
+            num_versions: 1,
+            num_objects: 1,
+        });
+        stats.add_stats(TierStats {
+            total_size: 20,
+            num_versions: 2,
+            num_objects: 0,
+        });
+
+        assert_eq!(
+            stats.total(),
+            TierStats {
+                total_size: 30,
+                num_versions: 3,
+                num_objects: 1,
+            }
+        );
+    }
+}

crates/ecstore/src/bucket/metadata.rs

@@ -33,7 +33,7 @@ use serde::Serializer;
 use std::collections::HashMap;
 use std::io::{Read, Write};
 use std::sync::Arc;
-use time::OffsetDateTime;
+use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset};
 use tracing::error;
 
 fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
@@ -43,19 +43,177 @@ fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
     Ok(String::from_utf8(buf)?)
 }
 
+fn read_msgp_bool<R: Read>(rd: &mut R) -> Result<bool> {
+    let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    match marker {
+        rmp::Marker::True => Ok(true),
+        rmp::Marker::False => Ok(false),
+        rmp::Marker::FixPos(v) => Ok(v != 0),
+        rmp::Marker::U8 => Ok(read_u8(rd)? != 0),
+        rmp::Marker::U16 => Ok(read_u16_raw(rd)? != 0),
+        rmp::Marker::U32 => Ok(read_u32_raw(rd)? != 0),
+        rmp::Marker::U64 => Ok(read_u64_raw(rd)? != 0),
+        rmp::Marker::I8 => Ok(read_i8_raw(rd)? != 0),
+        rmp::Marker::I16 => Ok(read_i16_raw(rd)? != 0),
+        rmp::Marker::I32 => Ok(read_i32_raw(rd)? != 0),
+        rmp::Marker::I64 => Ok(read_i64_raw(rd)? != 0),
+        rmp::Marker::FixNeg(v) => Ok(v != 0),
+        _ => Err(Error::other(format!("expected bool or int-like bool, got marker: {marker:?}"))),
+    }
+}
+
 fn read_msgp_time_value<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
     let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
     match marker {
         rmp::Marker::Null => Ok(OffsetDateTime::UNIX_EPOCH),
         rmp::Marker::Ext8 => read_msgp_ext8_time(rd),
+        rmp::Marker::FixArray(len) => read_msgp_legacy_compact_time(rd, u32::from(len)),
+        rmp::Marker::Array16 => {
+            let len = read_u16_raw(rd)?;
+            read_msgp_legacy_compact_time(rd, u32::from(len))
+        }
+        rmp::Marker::Array32 => {
+            let len = read_u32_raw(rd)?;
+            read_msgp_legacy_compact_time(rd, len)
+        }
+        rmp::Marker::Bin8 => {
+            let len = usize::from(read_u8(rd)?);
+            read_msgp_time_value_from_embedded_bin(rd, len)
+        }
+        rmp::Marker::Bin16 => {
+            let len = usize::from(read_u16_raw(rd)?);
+            read_msgp_time_value_from_embedded_bin(rd, len)
+        }
+        rmp::Marker::Bin32 => {
+            let len = read_u32_raw(rd)? as usize;
+            read_msgp_time_value_from_embedded_bin(rd, len)
+        }
         _ => Err(Error::other(format!("expected time ext or nil, got marker: {marker:?}"))),
     }
 }
 
-fn read_msgp_bin<R: Read>(rd: &mut R) -> Result<Vec<u8>> {
-    let len = rmp::decode::read_bin_len(rd)? as usize;
+fn read_u8<R: Read>(rd: &mut R) -> Result<u8> {
+    let mut buf = [0u8; 1];
+    rd.read_exact(&mut buf)?;
+    Ok(buf[0])
+}
+
+fn read_u16_raw<R: Read>(rd: &mut R) -> Result<u16> {
+    let mut buf = [0u8; 2];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_u16(&buf))
+}
+
+fn read_u32_raw<R: Read>(rd: &mut R) -> Result<u32> {
+    let mut buf = [0u8; 4];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_u32(&buf))
+}
+
+fn read_u64_raw<R: Read>(rd: &mut R) -> Result<u64> {
+    let mut buf = [0u8; 8];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_u64(&buf))
+}
+
+fn read_i8_raw<R: Read>(rd: &mut R) -> Result<i8> {
+    Ok(read_u8(rd)? as i8)
+}
+
+fn read_i16_raw<R: Read>(rd: &mut R) -> Result<i16> {
+    let mut buf = [0u8; 2];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_i16(&buf))
+}
+
+fn read_i32_raw<R: Read>(rd: &mut R) -> Result<i32> {
+    let mut buf = [0u8; 4];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_i32(&buf))
+}
+
+fn read_i64_raw<R: Read>(rd: &mut R) -> Result<i64> {
+    let mut buf = [0u8; 8];
+    rd.read_exact(&mut buf)?;
+    Ok(BigEndian::read_i64(&buf))
+}
+
+fn read_msgp_time_value_from_embedded_bin<R: Read>(rd: &mut R, len: usize) -> Result<OffsetDateTime> {
     let mut buf = vec![0u8; len];
     rd.read_exact(&mut buf)?;
+    let mut cur = std::io::Cursor::new(buf);
+    read_msgp_time_value(&mut cur)
+}
+
+fn read_msgp_legacy_compact_time<R: Read>(rd: &mut R, len: u32) -> Result<OffsetDateTime> {
+    if len != 9 {
+        return Err(Error::other(format!("invalid legacy compact time len: {len}")));
+    }
+
+    let year: i32 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let ordinal: u16 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let hour: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let minute: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let second: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let nanosecond: u32 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let offset_hour: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let offset_minute: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    let offset_second: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+
+    let date =
+        Date::from_ordinal_date(year, ordinal).map_err(|e| Error::other(format!("invalid legacy compact time date: {e}")))?;
+    let time = CivilTime::from_hms_nano(hour, minute, second, nanosecond)
+        .map_err(|e| Error::other(format!("invalid legacy compact time time: {e}")))?;
+    let offset = UtcOffset::from_hms(offset_hour, offset_minute, offset_second)
+        .map_err(|e| Error::other(format!("invalid legacy compact time offset: {e}")))?;
+
+    Ok(PrimitiveDateTime::new(date, time)
+        .assume_offset(offset)
+        .to_offset(UtcOffset::UTC))
+}
+
+fn read_msgp_bin<R: Read>(rd: &mut R) -> Result<Vec<u8>> {
+    let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+    match marker {
+        rmp::Marker::Null => Ok(Vec::new()),
+        rmp::Marker::Bin8 => {
+            let len = usize::from(read_u8(rd)?);
+            read_exact_bytes(rd, len)
+        }
+        rmp::Marker::Bin16 => {
+            let len = usize::from(read_u16_raw(rd)?);
+            read_exact_bytes(rd, len)
+        }
+        rmp::Marker::Bin32 => {
+            let len = read_u32_raw(rd)? as usize;
+            read_exact_bytes(rd, len)
+        }
+        rmp::Marker::FixArray(len) => read_msgp_legacy_byte_array(rd, u32::from(len)),
+        rmp::Marker::Array16 => {
+            let len = read_u16_raw(rd)?;
+            read_msgp_legacy_byte_array(rd, u32::from(len))
+        }
+        rmp::Marker::Array32 => {
+            let len = read_u32_raw(rd)?;
+            read_msgp_legacy_byte_array(rd, len)
+        }
+        _ => Err(Error::other(format!("expected bin or byte array, got marker: {marker:?}"))),
+    }
+}
+
+fn read_exact_bytes<R: Read>(rd: &mut R, len: usize) -> Result<Vec<u8>> {
+    let mut buf = vec![0u8; len];
+    rd.read_exact(&mut buf)?;
+    Ok(buf)
+}
+
+fn read_msgp_legacy_byte_array<R: Read>(rd: &mut R, len: u32) -> Result<Vec<u8>> {
+    let mut buf = Vec::with_capacity(len as usize);
+    for _ in 0..len {
+        let value: i64 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
+        let byte = u8::try_from(value).map_err(|_| Error::other(format!("byte value out of range: {value}")))?;
+        buf.push(byte);
+    }
     Ok(buf)
 }
 
@@ -227,18 +385,20 @@ impl BucketMetadata {
             match key.as_str() {
                 "Name" => self.name = read_msgp_str(rd)?,
                 "Created" => self.created = read_msgp_time_value(rd)?,
-                "LockEnabled" => self.lock_enabled = rmp::decode::read_bool(rd)?,
-                "PolicyConfigJSON" => self.policy_config_json = read_msgp_bin(rd)?,
-                "NotificationConfigXML" => self.notification_config_xml = read_msgp_bin(rd)?,
-                "LifecycleConfigXML" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
-                "ObjectLockConfigXML" => self.object_lock_config_xml = read_msgp_bin(rd)?,
-                "VersioningConfigXML" => self.versioning_config_xml = read_msgp_bin(rd)?,
-                "EncryptionConfigXML" => self.encryption_config_xml = read_msgp_bin(rd)?,
-                "TaggingConfigXML" => self.tagging_config_xml = read_msgp_bin(rd)?,
-                "QuotaConfigJSON" => self.quota_config_json = read_msgp_bin(rd)?,
-                "ReplicationConfigXML" => self.replication_config_xml = read_msgp_bin(rd)?,
-                "BucketTargetsConfigJSON" => self.bucket_targets_config_json = read_msgp_bin(rd)?,
-                "BucketTargetsConfigMetaJSON" => self.bucket_targets_config_meta_json = read_msgp_bin(rd)?,
+                "LockEnabled" => self.lock_enabled = read_msgp_bool(rd)?,
+                "PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?,
+                "NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?,
+                "LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
+                "ObjectLockConfigXML" | "ObjectLockConfigXml" => self.object_lock_config_xml = read_msgp_bin(rd)?,
+                "VersioningConfigXML" | "VersioningConfigXml" => self.versioning_config_xml = read_msgp_bin(rd)?,
+                "EncryptionConfigXML" | "EncryptionConfigXml" => self.encryption_config_xml = read_msgp_bin(rd)?,
+                "TaggingConfigXML" | "TaggingConfigXml" => self.tagging_config_xml = read_msgp_bin(rd)?,
+                "QuotaConfigJSON" | "QuotaConfigJson" => self.quota_config_json = read_msgp_bin(rd)?,
+                "ReplicationConfigXML" | "ReplicationConfigXml" => self.replication_config_xml = read_msgp_bin(rd)?,
+                "BucketTargetsConfigJSON" | "BucketTargetsConfigJson" => self.bucket_targets_config_json = read_msgp_bin(rd)?,
+                "BucketTargetsConfigMetaJSON" | "BucketTargetsConfigMetaJson" => {
+                    self.bucket_targets_config_meta_json = read_msgp_bin(rd)?
+                }
                 "PolicyConfigUpdatedAt" => self.policy_config_updated_at = read_msgp_time_value(rd)?,
                 "ObjectLockConfigUpdatedAt" => self.object_lock_config_updated_at = read_msgp_time_value(rd)?,
                 "EncryptionConfigUpdatedAt" => self.encryption_config_updated_at = read_msgp_time_value(rd)?,
@@ -250,9 +410,11 @@ impl BucketMetadata {
                 "NotificationConfigUpdatedAt" => self.notification_config_updated_at = read_msgp_time_value(rd)?,
                 "BucketTargetsConfigUpdatedAt" => self.bucket_targets_config_updated_at = read_msgp_time_value(rd)?,
                 "BucketTargetsConfigMetaUpdatedAt" => self.bucket_targets_config_meta_updated_at = read_msgp_time_value(rd)?,
-                "CorsConfigXML" => self.cors_config_xml = read_msgp_bin(rd)?,
-                "PublicAccessBlockConfigXML" => self.public_access_block_config_xml = read_msgp_bin(rd)?,
-                "BucketAclConfigJSON" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
+                "CorsConfigXML" | "CorsConfigXml" => self.cors_config_xml = read_msgp_bin(rd)?,
+                "PublicAccessBlockConfigXML" | "PublicAccessBlockConfigXml" => {
+                    self.public_access_block_config_xml = read_msgp_bin(rd)?
+                }
+                "BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
                 "CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
                 "PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?,
                 "BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,

crates/ecstore/src/bucket/metadata_test.rs

@@ -98,6 +98,143 @@ async fn unmarshal_test_bucket_metadata() {
     assert!(bm.bucket_acl_config_json.is_empty());
 }
 
+#[test]
+fn unmarshal_legacy_compact_time_bucket_metadata() {
+    use faster_hex::hex_decode;
+
+    let legacy_hex = concat!(
+        "83",
+        "a44e616d65",
+        "a474657374",
+        "a743726561746564",
+        "99cd07e9cd01100c1021ce2026b1fa000000",
+        "ab4c6f636b456e61626c6564",
+        "c2"
+    );
+
+    let mut bytes = vec![0u8; legacy_hex.len() / 2];
+    hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
+
+    let bm = BucketMetadata::unmarshal(&bytes).expect("legacy compact time should decode");
+
+    assert_eq!(bm.name, "test");
+    assert_eq!(bm.created.unix_timestamp(), 1759148193);
+    assert_eq!(bm.created.nanosecond(), 539406842);
+    assert!(!bm.lock_enabled);
+}
+
+#[test]
+fn unmarshal_bin_wrapped_ext_time_bucket_metadata() {
+    use faster_hex::hex_decode;
+
+    let wrapped_hex = concat!(
+        "83",
+        "a44e616d65",
+        "a464616461",
+        "a743726561746564",
+        "c40fc70c05fffffff1886e090000000000",
+        "ab4c6f636b456e61626c6564",
+        "c2"
+    );
+
+    let mut bytes = vec![0u8; wrapped_hex.len() / 2];
+    hex_decode(wrapped_hex.as_bytes(), &mut bytes).expect("valid hex");
+
+    let bm = BucketMetadata::unmarshal(&bytes).expect("bin-wrapped ext time should decode");
+
+    assert_eq!(bm.name, "dada");
+    assert_eq!(bm.created.unix_timestamp(), -62135596800);
+    assert!(!bm.lock_enabled);
+}
+
+#[test]
+fn unmarshal_legacy_rmp_serde_field_aliases_and_byte_arrays() {
+    use faster_hex::hex_decode;
+
+    let legacy_hex = concat!(
+        "85",
+        "a44e616d65",
+        "a474657374",
+        "a743726561746564",
+        "99cd07e9cd01100c1021ce2026b1fa000000",
+        "ab4c6f636b456e61626c6564",
+        "c2",
+        "b0506f6c696379436f6e6669674a736f6e",
+        "93010203",
+        "bb4275636b657454617267657473436f6e6669674d6574614a736f6e",
+        "920405"
+    );
+
+    let mut bytes = vec![0u8; legacy_hex.len() / 2];
+    hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
+
+    let bm = BucketMetadata::unmarshal(&bytes).expect("legacy field aliases and byte arrays should decode");
+
+    assert_eq!(bm.name, "test");
+    assert_eq!(bm.created.unix_timestamp(), 1759148193);
+    assert_eq!(bm.created.nanosecond(), 539406842);
+    assert_eq!(bm.policy_config_json, vec![1, 2, 3]);
+    assert_eq!(bm.bucket_targets_config_meta_json, vec![4, 5]);
+    assert!(!bm.lock_enabled);
+}
+
+#[test]
+fn unmarshal_legacy_bin16_and_array16_bucket_metadata() {
+    let policy = vec![b'x'; 257];
+    let targets_meta: Vec<u8> = (0u8..=16).collect();
+
+    let mut bytes = Vec::new();
+    rmp::encode::write_map_len(&mut bytes, 5).unwrap();
+    rmp::encode::write_str(&mut bytes, "Name").unwrap();
+    rmp::encode::write_str(&mut bytes, "test-bucket").unwrap();
+    rmp::encode::write_str(&mut bytes, "Created").unwrap();
+    bytes.extend_from_slice(&[
+        0xc7, 0x0c, 0x05, 0x00, 0x00, 0x00, 0x00, 0x65, 0x92, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
+    ]);
+    rmp::encode::write_str(&mut bytes, "LockEnabled").unwrap();
+    bytes.push(0x01);
+    rmp::encode::write_str(&mut bytes, "PolicyConfigJson").unwrap();
+    rmp::encode::write_bin(&mut bytes, &policy).unwrap();
+    rmp::encode::write_str(&mut bytes, "BucketTargetsConfigMetaJson").unwrap();
+    rmp::encode::write_array_len(&mut bytes, targets_meta.len() as u32).unwrap();
+    for byte in &targets_meta {
+        rmp::encode::write_uint(&mut bytes, u64::from(*byte)).unwrap();
+    }
+
+    let bm = BucketMetadata::unmarshal(&bytes).expect("legacy bin16 and array16 should decode");
+
+    assert_eq!(bm.name, "test-bucket");
+    assert_eq!(bm.created.unix_timestamp(), 1704067200);
+    assert!(bm.lock_enabled);
+    assert_eq!(bm.policy_config_json, policy);
+    assert_eq!(bm.bucket_targets_config_meta_json, targets_meta);
+}
+
+#[test]
+fn unmarshal_legacy_numeric_bool_bucket_metadata() {
+    use faster_hex::hex_decode;
+
+    let legacy_hex = concat!(
+        "83",
+        "a44e616d65",
+        "ab746573742d6275636b6574",
+        "a743726561746564",
+        "c70c05000000006592008000000000",
+        "ab4c6f636b456e61626c6564",
+        "01"
+    );
+
+    let mut bytes = vec![0u8; legacy_hex.len() / 2];
+    hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
+
+    let bm = BucketMetadata::unmarshal(&bytes).expect("legacy numeric bool should decode");
+
+    assert_eq!(bm.name, "test-bucket");
+    assert_eq!(bm.created.unix_timestamp(), 1704067200);
+    assert_eq!(bm.created.nanosecond(), 0);
+    assert!(bm.lock_enabled);
+}
+
 #[tokio::test]
 async fn marshal_msg_complete_example() {
     // Create a complete BucketMetadata with various configurations

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2453,7 +2453,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             return rinfo;
         }
 
-        let sopts = StatObjectOptions {
+        let mut sopts = StatObjectOptions {
             version_id: object_info.version_id.map(|v| v.to_string()).unwrap_or_default(),
             internal: AdvancedGetOptions {
                 replication_proxy_request: "false".to_string(),
@@ -2462,7 +2462,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             ..Default::default()
         };
 
-        sopts.set(AMZ_TAGGING_DIRECTIVE, "ACCESS");
+        if let Err(err) = sopts.set(AMZ_TAGGING_DIRECTIVE, "ACCESS") {
+            warn!("failed to set replication tagging directive header: {err}");
+        }
 
         match tgt_client
             .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))

crates/ecstore/src/client/api_get_options.rs

@@ -64,20 +64,32 @@ impl GetObjectOptions {
     pub fn header(&self) -> HeaderMap {
         let mut headers: HeaderMap = HeaderMap::with_capacity(self.headers.len());
         for (k, v) in &self.headers {
-            if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
-                headers.insert(header_name, v.parse().expect("err"));
-            } else {
-                warn!("Invalid header name: {}", k);
+            match (HeaderName::from_bytes(k.as_bytes()), HeaderValue::from_str(v)) {
+                (Ok(header_name), Ok(header_value)) => {
+                    headers.insert(header_name, header_value);
+                }
+                (Err(_), _) => {
+                    warn!("Invalid header name: {}", k);
+                }
+                (_, Err(_)) => {
+                    warn!("Invalid header value for {}: {:?}", k, v);
+                }
             }
         }
         if self.checksum {
-            headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err"));
+            headers.insert(HeaderName::from_static("x-amz-checksum-mode"), HeaderValue::from_static("ENABLED"));
         }
         headers
     }
 
-    pub fn set(&self, key: &str, value: &str) {
-        //self.headers[http.CanonicalHeaderKey(key)] = value;
+    pub fn set(&mut self, key: &str, value: &str) -> Result<(), std::io::Error> {
+        let header_name = HeaderName::from_bytes(key.as_bytes())
+            .map_err(|err| std::io::Error::other(err_invalid_argument(&format!("Invalid header name {key}: {err}"))))?;
+        HeaderValue::from_str(value)
+            .map_err(|err| std::io::Error::other(err_invalid_argument(&format!("Invalid header value for {key}: {err}"))))?;
+
+        self.headers.insert(header_name.as_str().to_string(), value.to_string());
+        Ok(())
     }
 
     pub fn set_req_param(&mut self, key: &str, value: &str) {
@@ -89,12 +101,12 @@ impl GetObjectOptions {
     }
 
     pub fn set_match_etag(&mut self, etag: &str) -> Result<(), std::io::Error> {
-        self.set("If-Match", &format!("\"{etag}\""));
+        self.set("If-Match", &format!("\"{etag}\""))?;
         Ok(())
     }
 
     pub fn set_match_etag_except(&mut self, etag: &str) -> Result<(), std::io::Error> {
-        self.set("If-None-Match", &format!("\"{etag}\""));
+        self.set("If-None-Match", &format!("\"{etag}\""))?;
         Ok(())
     }
 
@@ -102,7 +114,7 @@ impl GetObjectOptions {
         if mod_time.unix_timestamp() == 0 {
             return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
         }
-        self.set("If-Unmodified-Since", &mod_time.to_string());
+        self.set("If-Unmodified-Since", &mod_time.to_string())?;
         Ok(())
     }
 
@@ -110,17 +122,17 @@ impl GetObjectOptions {
         if mod_time.unix_timestamp() == 0 {
             return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
         }
-        self.set("If-Modified-Since", &mod_time.to_string());
+        self.set("If-Modified-Since", &mod_time.to_string())?;
         Ok(())
     }
 
     pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> {
         if start == 0 && end < 0 {
-            self.set("Range", &format!("bytes={}", end));
+            self.set("Range", &format!("bytes={}", end))?;
         } else if 0 < start && end == 0 {
-            self.set("Range", &format!("bytes={}-", start));
+            self.set("Range", &format!("bytes={}-", start))?;
         } else if 0 <= start && start <= end {
-            self.set("Range", &format!("bytes={}-{}", start, end));
+            self.set("Range", &format!("bytes={}-{}", start, end))?;
         } else {
             return Err(std::io::Error::other(err_invalid_argument(&format!(
                 "Invalid range specified: start={} end={}",
@@ -146,3 +158,40 @@ impl GetObjectOptions {
         url_values
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::GetObjectOptions;
+
+    #[test]
+    fn set_range_populates_range_header() {
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(5, 9).expect("valid range should succeed");
+
+        let headers = opts.header();
+        let range = headers.get("range").expect("range header should be present");
+        assert_eq!(range.to_str().expect("range header must be valid ascii"), "bytes=5-9");
+    }
+
+    #[test]
+    fn set_rejects_invalid_header_value() {
+        let mut opts = GetObjectOptions::default();
+
+        let err = opts
+            .set("Range", "bytes=5-\n9")
+            .expect_err("invalid header value should fail");
+
+        assert!(err.to_string().contains("Invalid header value"));
+        assert!(opts.headers.is_empty(), "invalid headers must not be stored");
+    }
+
+    #[test]
+    fn header_skips_invalid_prepopulated_header_value() {
+        let mut opts = GetObjectOptions::default();
+        opts.headers.insert("Range".to_string(), "bytes=5-\n9".to_string());
+
+        let headers = opts.header();
+
+        assert!(headers.get("range").is_none(), "invalid stored header values should be ignored");
+    }
+}

crates/ecstore/src/client/transition_api.rs

@@ -375,7 +375,7 @@ impl TransitionClient {
         //debug!("http_resp_body: {}", String::from_utf8(b).unwrap());
 
         //if self.is_trace_enabled && !(self.trace_errors_only && resp.status() == StatusCode::OK) {
-        if resp.status() != StatusCode::OK {
+        if !resp.status().is_success() {
             //self.dump_http(&cloned_req, &resp)?;
             let mut body_vec = Vec::new();
             let mut body = resp.into_body();

crates/ecstore/src/config/com.rs

@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::config::{Config, GLOBAL_STORAGE_CLASS, storageclass};
+use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, oidc, storageclass};
 use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result};
+use crate::global::is_first_cluster_node_local;
 use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
 use http::HeaderMap;
-use rustfs_config::{DEFAULT_DELIMITER, RUSTFS_REGION};
+use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
+use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
 use rustfs_utils::path::SLASH_SEPARATOR;
 use serde_json::{Map, Value};
 use std::collections::{HashMap, HashSet};
@@ -44,6 +46,19 @@ pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u
     Ok(data)
 }
 
+pub async fn read_config_no_lock<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
+    let (data, _obj) = read_config_with_metadata(
+        api,
+        file,
+        &ObjectOptions {
+            no_lock: true,
+            ..Default::default()
+        },
+    )
+    .await?;
+    Ok(data)
+}
+
 pub async fn read_config_with_metadata<S: StorageAPI>(
     api: Arc<S>,
     file: &str,
@@ -167,6 +182,85 @@ fn parse_inline_block_value(value: &Value) -> Option<String> {
     }
 }
 
+fn parse_oidc_scalar_value(key: &str, value: &Value) -> Option<String> {
+    match value {
+        Value::String(v) => Some(v.trim().to_string()),
+        Value::Bool(v) if key == ENABLE_KEY || key == OIDC_REDIRECT_URI_DYNAMIC => Some(if *v {
+            EnableState::On.to_string()
+        } else {
+            EnableState::Off.to_string()
+        }),
+        Value::Bool(v) => Some(v.to_string()),
+        Value::Number(v) => Some(v.to_string()),
+        Value::Array(values) if key == rustfs_config::oidc::OIDC_SCOPES => {
+            let scopes = values
+                .iter()
+                .filter_map(Value::as_str)
+                .map(str::trim)
+                .filter(|scope| !scope.is_empty())
+                .collect::<Vec<_>>()
+                .join(",");
+            Some(scopes)
+        }
+        Value::Null => None,
+        _ => None,
+    }
+}
+
+fn decode_oidc_provider_object(provider: &Map<String, Value>) -> KVS {
+    let mut kvs = oidc::DEFAULT_IDENTITY_OPENID_KVS.clone();
+
+    for (key, value) in provider {
+        if !IDENTITY_OPENID_KEYS.contains(&key.as_str()) || key == COMMENT_KEY {
+            continue;
+        }
+
+        if let Some(parsed) = parse_oidc_scalar_value(key, value) {
+            kvs.insert(key.clone(), parsed);
+        }
+    }
+
+    kvs
+}
+
+fn apply_external_oidc_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
+    let oidc_root = root.get("openid").or_else(|| root.get(IDENTITY_OPENID_SUB_SYS));
+    let Some(Value::Object(oidc_obj)) = oidc_root else {
+        return false;
+    };
+
+    if oidc_obj.is_empty() {
+        return false;
+    }
+
+    let subsystem = cfg.0.entry(IDENTITY_OPENID_SUB_SYS.to_string()).or_default();
+    let mut applied = false;
+
+    for (raw_instance, provider) in oidc_obj {
+        let instance_key = if raw_instance == "default" {
+            DEFAULT_DELIMITER.to_string()
+        } else {
+            raw_instance.to_string()
+        };
+
+        match provider {
+            Value::Object(provider_obj) => {
+                subsystem.insert(instance_key, decode_oidc_provider_object(provider_obj));
+                applied = true;
+            }
+            Value::Array(_) => {
+                if let Ok(kvs) = serde_json::from_value::<KVS>(provider.clone()) {
+                    subsystem.insert(instance_key, kvs);
+                    applied = true;
+                }
+            }
+            _ => {}
+        }
+    }
+
+    applied
+}
+
 fn apply_external_storage_class_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
     let sc = root.get("storageclass").or_else(|| root.get("storage_class"));
     let Some(Value::Object(sc_obj)) = sc else {
@@ -210,8 +304,9 @@ fn decode_server_config_blob(data: &[u8]) -> Result<Config> {
 
     let mut cfg = Config::new();
     let has_storage = apply_external_storage_class_map(&mut cfg, &root);
+    let has_oidc = apply_external_oidc_map(&mut cfg, &root);
     let has_header = root.contains_key("version") || root.contains_key("region") || root.contains_key("credential");
-    if !has_storage && !has_header {
+    if !has_storage && !has_oidc && !has_header {
         return Err(Error::other("unrecognized external server config shape"));
     }
     Ok(cfg)
@@ -241,6 +336,119 @@ fn build_storageclass_object(cfg: &Config) -> Map<String, Value> {
     sc_obj
 }
 
+fn build_oidc_provider_object(kvs: &KVS) -> Map<String, Value> {
+    let mut provider = Map::new();
+
+    for kv in &kvs.0 {
+        if kv.key == COMMENT_KEY || (kv.hidden_if_empty && kv.value.trim().is_empty()) {
+            continue;
+        }
+
+        if kv.value.trim().is_empty() {
+            continue;
+        }
+
+        if kv.key == ENABLE_KEY || kv.key == OIDC_REDIRECT_URI_DYNAMIC {
+            let enabled = kv
+                .value
+                .parse::<EnableState>()
+                .map(|state| state.is_enabled())
+                .unwrap_or(false);
+            provider.insert(kv.key.clone(), Value::Bool(enabled));
+            continue;
+        }
+
+        if kv.key == rustfs_config::oidc::OIDC_SCOPES {
+            let scopes = kv
+                .value
+                .split(',')
+                .map(str::trim)
+                .filter(|scope| !scope.is_empty())
+                .map(|scope| Value::String(scope.to_string()))
+                .collect::<Vec<_>>();
+            provider.insert(kv.key.clone(), Value::Array(scopes));
+            continue;
+        }
+
+        provider.insert(kv.key.clone(), Value::String(kv.value.clone()));
+    }
+
+    provider
+}
+
+fn build_oidc_object(cfg: &Config) -> Map<String, Value> {
+    let Some(subsystem) = cfg.0.get(IDENTITY_OPENID_SUB_SYS) else {
+        return Map::new();
+    };
+
+    let mut providers = subsystem.iter().collect::<Vec<_>>();
+    providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+
+    let mut oidc_obj = Map::new();
+    for (instance_key, kvs) in providers {
+        if kvs
+            .lookup(rustfs_config::oidc::OIDC_CONFIG_URL)
+            .unwrap_or_default()
+            .trim()
+            .is_empty()
+        {
+            continue;
+        }
+
+        let provider = build_oidc_provider_object(kvs);
+        if provider.is_empty() {
+            continue;
+        }
+
+        let external_key = if instance_key == DEFAULT_DELIMITER {
+            "default".to_string()
+        } else {
+            instance_key.clone()
+        };
+        oidc_obj.insert(external_key, Value::Object(provider));
+    }
+
+    oidc_obj
+}
+
+fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
+    let Some(subsystem) = cfg.0.get(IDENTITY_OPENID_SUB_SYS) else {
+        return Map::new();
+    };
+
+    let mut providers = subsystem.iter().collect::<Vec<_>>();
+    providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+
+    let mut oidc_obj = Map::new();
+    for (instance_key, kvs) in providers {
+        let mut normalized = oidc::DEFAULT_IDENTITY_OPENID_KVS.clone();
+        normalized.extend(kvs.clone());
+
+        if normalized
+            .lookup(rustfs_config::oidc::OIDC_CONFIG_URL)
+            .unwrap_or_default()
+            .trim()
+            .is_empty()
+        {
+            continue;
+        }
+
+        let provider = build_oidc_provider_object(&normalized);
+        if provider.is_empty() {
+            continue;
+        }
+
+        let external_key = if instance_key == DEFAULT_DELIMITER {
+            "default".to_string()
+        } else {
+            instance_key.clone()
+        };
+        oidc_obj.insert(external_key, Value::Object(provider));
+    }
+
+    oidc_obj
+}
+
 fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8>> {
     let mut root = seed.and_then(parse_object_seed).unwrap_or_default();
 
@@ -261,6 +469,15 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
     root.insert("storageclass".to_string(), Value::Object(sc_obj));
     root.remove("storage_class");
 
+    let oidc_obj = build_oidc_object(cfg);
+    if oidc_obj.is_empty() {
+        root.remove("openid");
+        root.remove(IDENTITY_OPENID_SUB_SYS);
+    } else {
+        root.insert("openid".to_string(), Value::Object(oidc_obj));
+        root.remove(IDENTITY_OPENID_SUB_SYS);
+    }
+
     Ok(serde_json::to_vec(&Value::Object(root))?)
 }
 
@@ -278,6 +495,7 @@ fn is_standard_object_server_config(data: &[u8]) -> bool {
 
 fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool {
     build_storageclass_object(lhs) == build_storageclass_object(rhs)
+        && build_semantic_oidc_object(lhs) == build_semantic_oidc_object(rhs)
 }
 
 fn is_object_not_found(err: &Error) -> bool {
@@ -287,7 +505,14 @@ fn is_object_not_found(err: &Error) -> bool {
 pub async fn try_migrate_server_config<S: StorageAPI>(api: Arc<S>) {
     let config_file = get_config_file();
     match api
-        .get_object_info(RUSTFS_META_BUCKET, &config_file, &ObjectOptions::default())
+        .get_object_info(
+            RUSTFS_META_BUCKET,
+            &config_file,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
         .await
     {
         Ok(_) => {
@@ -360,7 +585,13 @@ pub async fn try_migrate_server_config<S: StorageAPI>(api: Arc<S>) {
 /// Handle the situation where the configuration file does not exist, create and save a new configuration
 async fn handle_missing_config<S: StorageAPI>(api: Arc<S>, context: &str) -> Result<Config> {
     warn!("Configuration not found ({}): Start initializing new configuration", context);
-    let cfg = new_and_save_server_config(api).await?;
+    let cfg = if is_first_cluster_node_local().await {
+        new_and_save_server_config(api.clone()).await?
+    } else {
+        let mut cfg = new_server_config();
+        lookup_configs(&mut cfg, api).await;
+        cfg
+    };
     warn!("Configuration initialization complete ({})", context);
     Ok(cfg)
 }
@@ -375,7 +606,7 @@ pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<C
     let config_file = get_config_file();
 
     // Try to read the configuration file
-    match read_config(api.clone(), &config_file).await {
+    match read_config_no_lock(api.clone(), &config_file).await {
         Ok(data) => read_server_config(api, &data).await,
         Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await,
         Err(err) => handle_config_read_error(err, &config_file),
@@ -389,7 +620,7 @@ async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<C
         warn!("Received empty configuration data, try to reread from '{}'", config_file);
 
         // Try to read the configuration again
-        match read_config(api.clone(), &config_file).await {
+        match read_config_no_lock(api.clone(), &config_file).await {
             Ok(cfg_data) => {
                 // TODO: decrypt
                 let cfg = decode_server_config_blob(&cfg_data)?;
@@ -481,7 +712,9 @@ mod tests {
         configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config,
         storage_class_kvs_mut,
     };
-    use crate::config::Config;
+    use crate::config::{Config, oidc};
+    use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
+    use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
     use serde_json::Value;
 
     #[test]
@@ -523,6 +756,54 @@ mod tests {
         assert_eq!(kvs.get("optimize"), "availability");
     }
 
+    #[test]
+    fn test_decode_server_config_reads_openid_providers() {
+        let input = r#"{
+          "version":"33",
+          "storageclass":{"standard":"EC:2","rrs":"EC:1"},
+          "openid":{
+            "default":{
+              "enable":true,
+              "config_url":"https://example.com/.well-known/openid-configuration",
+              "client_id":"console",
+              "client_secret":"secret-value",
+              "scopes":["openid","profile","email"],
+              "redirect_uri_dynamic":true,
+              "display_name":"Default Provider"
+            },
+            "smoke":{
+              "enable":false,
+              "config_url":"https://issuer.example.com/.well-known/openid-configuration",
+              "client_id":"smoke-client",
+              "scopes":["openid"],
+              "redirect_uri_dynamic":false
+            }
+          }
+        }"#;
+
+        let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
+
+        let default_kvs = cfg
+            .get_value(IDENTITY_OPENID_SUB_SYS, DEFAULT_DELIMITER)
+            .expect("default oidc provider should exist");
+        assert_eq!(
+            default_kvs.get(rustfs_config::oidc::OIDC_CONFIG_URL),
+            "https://example.com/.well-known/openid-configuration"
+        );
+        assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_CLIENT_ID), "console");
+        assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_SCOPES), "openid,profile,email");
+        assert_eq!(default_kvs.get(ENABLE_KEY), EnableState::On.to_string());
+
+        let smoke_kvs = cfg
+            .get_value(IDENTITY_OPENID_SUB_SYS, "smoke")
+            .expect("named oidc provider should exist");
+        assert_eq!(smoke_kvs.get(rustfs_config::oidc::OIDC_CLIENT_ID), "smoke-client");
+        assert_eq!(
+            smoke_kvs.get(rustfs_config::oidc::OIDC_REDIRECT_URI_DYNAMIC),
+            EnableState::Off.to_string()
+        );
+    }
+
     #[test]
     fn test_encode_server_config_writes_external_object_shape() {
         let mut cfg = Config::new();
@@ -537,6 +818,48 @@ mod tests {
         assert!(v.get("storage_class").is_none(), "should not write rustfs map shape");
     }
 
+    #[test]
+    fn test_encode_server_config_writes_openid_object_shape() {
+        let mut cfg = Config::new();
+        let mut oidc_section = std::collections::HashMap::new();
+        let mut default_provider = oidc::DEFAULT_IDENTITY_OPENID_KVS.clone();
+        default_provider.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        default_provider.insert(
+            rustfs_config::oidc::OIDC_CONFIG_URL.to_string(),
+            "https://example.com/.well-known/openid-configuration".to_string(),
+        );
+        default_provider.insert(rustfs_config::oidc::OIDC_CLIENT_ID.to_string(), "console".to_string());
+        default_provider.insert(rustfs_config::oidc::OIDC_SCOPES.to_string(), "openid,profile,email".to_string());
+        oidc_section.insert(DEFAULT_DELIMITER.to_string(), default_provider);
+        cfg.0.insert(IDENTITY_OPENID_SUB_SYS.to_string(), oidc_section);
+
+        let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
+        let v: Value = serde_json::from_slice(&out).expect("output should be json");
+        let openid = v
+            .get("openid")
+            .and_then(Value::as_object)
+            .expect("output should include openid object");
+        let default_provider = openid
+            .get("default")
+            .and_then(Value::as_object)
+            .expect("default provider should be encoded");
+
+        assert_eq!(
+            default_provider
+                .get(rustfs_config::oidc::OIDC_CLIENT_ID)
+                .and_then(Value::as_str),
+            Some("console")
+        );
+        assert_eq!(
+            default_provider
+                .get(rustfs_config::oidc::OIDC_SCOPES)
+                .and_then(Value::as_array)
+                .map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
+            Some(vec!["openid", "profile", "email"])
+        );
+        assert_eq!(default_provider.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
+    }
+
     #[test]
     fn test_is_standard_object_server_config_detection() {
         let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"}}"#;
@@ -554,4 +877,39 @@ mod tests {
         let rhs = decode_server_config_blob(legacy).expect("decode legacy");
         assert!(configs_semantically_equal(&lhs, &rhs));
     }
+
+    #[test]
+    fn test_configs_semantically_equal_accounts_for_openid() {
+        let external = br#"{
+          "version":"33",
+          "storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"},
+          "openid":{
+            "default":{
+              "enable":true,
+              "config_url":"https://example.com/.well-known/openid-configuration",
+              "client_id":"console",
+              "scopes":["openid","profile","email"],
+              "redirect_uri_dynamic":true
+            }
+          }
+        }"#;
+        let legacy = br#"{
+          "storage_class":{"_":[
+            {"key":"standard","value":"EC:2"},
+            {"key":"rrs","value":"EC:1"},
+            {"key":"optimize","value":"availability"}
+          ]},
+          "identity_openid":{"_":[
+            {"key":"enable","value":"on"},
+            {"key":"config_url","value":"https://example.com/.well-known/openid-configuration"},
+            {"key":"client_id","value":"console"},
+            {"key":"scopes","value":"openid,profile,email"},
+            {"key":"redirect_uri_dynamic","value":"on"}
+          ]}
+        }"#;
+
+        let lhs = decode_server_config_blob(external).expect("decode external");
+        let rhs = decode_server_config_blob(legacy).expect("decode legacy");
+        assert!(configs_semantically_equal(&lhs, &rhs));
+    }
 }

crates/ecstore/src/config/mod.rs

@@ -79,7 +79,7 @@ pub async fn try_migrate_server_config(api: Arc<ECStore>) {
     com::try_migrate_server_config(api).await
 }
 
-#[derive(Debug, Deserialize, Serialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
 pub struct KV {
     pub key: String,
     pub value: String,
@@ -87,7 +87,7 @@ pub struct KV {
     pub hidden_if_empty: bool,
 }
 
-#[derive(Debug, Deserialize, Serialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
 pub struct KVS(pub Vec<KV>);
 
 impl Default for KVS {
@@ -163,7 +163,7 @@ impl KVS {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Config(pub HashMap<String, HashMap<String, KVS>>);
 
 impl Default for Config {

crates/ecstore/src/disk/disk_store.rs

@@ -18,6 +18,7 @@ use crate::disk::{
     WalkDirOptions,
     local::{LocalDisk, ScanGuard},
 };
+use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use std::{
@@ -410,7 +411,19 @@ impl LocalDiskWrapper {
     /// Set the disk ID
     pub async fn set_disk_id_internal(&self, id: Option<Uuid>) -> Result<()> {
         let mut disk_id = self.disk_id.write().await;
+        let previous = *disk_id;
         *disk_id = id;
+        drop(disk_id);
+
+        if self.disk.is_local() {
+            let mut disk_id_map = GLOBAL_LOCAL_DISK_ID_MAP.write().await;
+            if let Some(previous_id) = previous {
+                disk_id_map.remove(&previous_id);
+            }
+            if let Some(current_id) = id {
+                disk_id_map.insert(current_id, self.disk.endpoint().to_string());
+            }
+        }
         Ok(())
     }
 

crates/ecstore/src/disk/local.rs

@@ -29,7 +29,6 @@ use crate::disk::{
     os::{check_path_length, is_empty_dir, is_root_disk, rename_all},
 };
 use crate::erasure_coding::bitrot_verify;
-use crate::file_cache::{get_global_file_cache, prefetch_metadata_patterns, read_metadata_cached};
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
 use bytes::Bytes;
 use parking_lot::RwLock as ParkingLotRwLock;
@@ -492,56 +491,6 @@ impl LocalDisk {
         Ok(results.into_iter().map(|(_, path)| path).collect())
     }
 
-    // Optimized metadata reading with caching
-    async fn read_metadata_cached(&self, path: PathBuf) -> Result<Arc<FileMeta>> {
-        read_metadata_cached(path).await
-    }
-
-    // Smart prefetching for related files
-    async fn read_version_with_prefetch(
-        &self,
-        volume: &str,
-        path: &str,
-        version_id: &str,
-        opts: &ReadOptions,
-    ) -> Result<FileInfo> {
-        let file_path = self.get_object_path(volume, path)?;
-
-        // Async prefetch related files, don't block current read
-        if let Some(parent) = file_path.parent() {
-            prefetch_metadata_patterns(parent, &[STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await;
-        }
-
-        // Main read logic
-        let file_dir = self.get_bucket_path(volume)?;
-        let (data, _) = self.read_raw(volume, file_dir, file_path, opts.read_data).await?;
-
-        get_file_info(
-            &data,
-            volume,
-            path,
-            version_id,
-            FileInfoOpts {
-                data: opts.read_data,
-                include_free_versions: false,
-            },
-        )
-        .map_err(|_e| DiskError::Unexpected)
-    }
-
-    // Batch metadata reading for multiple objects
-    async fn read_metadata_batch(&self, requests: Vec<(String, String)>) -> Result<Vec<Option<Arc<FileMeta>>>> {
-        let paths: Vec<PathBuf> = requests
-            .iter()
-            .map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, STORAGE_FORMAT_FILE)))
-            .collect::<Result<Vec<_>>>()?;
-
-        let cache = get_global_file_cache();
-        let results = cache.get_metadata_batch(paths).await;
-
-        Ok(results.into_iter().map(|r| r.ok()).collect())
-    }
-
     // /// Write to the filesystem atomically.
     // /// This is done by first writing to a temporary location and then moving the file.
     // pub(crate) async fn prepare_file_write<'a>(&self, path: &'a PathBuf) -> Result<FileWriter<'a>> {
@@ -865,8 +814,6 @@ impl LocalDisk {
 
         rename_all(tmp_file_path, &file_path, volume_dir).await?;
 
-        // Invalidate cache after successful write
-        get_global_file_cache().invalidate(&file_path).await;
         Ok(())
     }
 
@@ -891,7 +838,9 @@ impl LocalDisk {
         check_path_length(file_path.to_string_lossy().as_ref())?;
 
         self.write_all_internal(&file_path, InternalBuf::Owned(buf), sync, skip_parent)
-            .await
+            .await?;
+
+        Ok(())
     }
     // write_all_internal do write file
     async fn write_all_internal(&self, file_path: &Path, data: InternalBuf<'_>, sync: bool, skip_parent: &Path) -> Result<()> {
@@ -2478,7 +2427,7 @@ impl DiskAPI for LocalDisk {
                 file_path.as_path(),
                 Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str()),
             ]);
-            return rename_all(src_path, dst_path, file_path).await;
+            return rename_all(&src_path, &dst_path, file_path).await;
         }
 
         self.delete_file(&volume_dir, &xl_path, true, false).await
@@ -2601,17 +2550,9 @@ impl DiskAPI for LocalDisk {
 
     #[tracing::instrument(skip(self))]
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
-        // Try to use cached file content reading for better performance, with safe fallback
         let file_path = self.get_object_path(volume, path)?;
-        // let file_path = file_path.join(Path::new(STORAGE_FORMAT_FILE));
-
-        // First, try the cache
-        if let Ok(bytes) = get_global_file_cache().get_file_content(file_path.clone()).await {
-            return Ok(bytes);
-        }
-
-        // Fallback to direct read if cache fails
-        let (data, _) = self.read_metadata_with_dmtime(&file_path).await?;
+        let volume_dir = self.get_bucket_path(volume)?;
+        let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
         Ok(data.into())
     }
 }

crates/ecstore/src/endpoints.rs

@@ -791,14 +791,17 @@ mod test {
             panic!("No non-loop back IP address found for this host");
         }
         let non_loop_back_ip = non_loop_back_i_ps[0];
+        let remote_ip1 = "192.0.2.10";
+        let remote_ip2 = "192.0.2.11";
+        let remote_ip3 = "192.0.2.12";
 
         let case1_endpoint1 = format!("http://{non_loop_back_ip}/d1");
         let case1_endpoint2 = format!("http://{non_loop_back_ip}/d2");
         let args = vec![
             format!("http://{}:10000/d1", non_loop_back_ip),
             format!("http://{}:10000/d2", non_loop_back_ip),
-            "http://example.org:10000/d3".to_string(),
-            "http://example.com:10000/d4".to_string(),
+            format!("http://{remote_ip1}:10000/d3"),
+            format!("http://{remote_ip2}:10000/d4"),
         ];
         let (case1_ur_ls, case1_local_flags) = get_expected_endpoints(args, format!("http://{non_loop_back_ip}:10000/"));
 
@@ -807,26 +810,26 @@ mod test {
         let args = vec![
             format!("http://{}:10000/d1", non_loop_back_ip),
             format!("http://{}:9000/d2", non_loop_back_ip),
-            "http://example.org:10000/d3".to_string(),
-            "http://example.com:10000/d4".to_string(),
+            format!("http://{remote_ip1}:10000/d3"),
+            format!("http://{remote_ip2}:10000/d4"),
         ];
         let (case2_ur_ls, case2_local_flags) = get_expected_endpoints(args, format!("http://{non_loop_back_ip}:10000/"));
 
         let case3_endpoint1 = format!("http://{non_loop_back_ip}/d1");
         let args = vec![
             format!("http://{}:80/d1", non_loop_back_ip),
-            "http://example.org:9000/d2".to_string(),
-            "http://example.com:80/d3".to_string(),
-            "http://example.net:80/d4".to_string(),
+            format!("http://{remote_ip1}:9000/d2"),
+            format!("http://{remote_ip2}:80/d3"),
+            format!("http://{remote_ip3}:80/d4"),
         ];
         let (case3_ur_ls, case3_local_flags) = get_expected_endpoints(args, format!("http://{non_loop_back_ip}:80/"));
 
         let case4_endpoint1 = format!("http://{non_loop_back_ip}/d1");
         let args = vec![
             format!("http://{}:9000/d1", non_loop_back_ip),
-            "http://example.org:9000/d2".to_string(),
-            "http://example.com:9000/d3".to_string(),
-            "http://example.net:9000/d4".to_string(),
+            format!("http://{remote_ip1}:9000/d2"),
+            format!("http://{remote_ip2}:9000/d3"),
+            format!("http://{remote_ip3}:9000/d4"),
         ];
         let (case4_ur_ls, case4_local_flags) = get_expected_endpoints(args, format!("http://{non_loop_back_ip}:9000/"));
 
@@ -844,8 +847,8 @@ mod test {
 
         let case6_endpoint1 = format!("http://{non_loop_back_ip}:9003/d4");
         let args = vec![
-            "http://localhost:9000/d1".to_string(),
-            "http://localhost:9001/d2".to_string(),
+            "http://127.0.0.1:9000/d1".to_string(),
+            "http://127.0.0.1:9001/d2".to_string(),
             "http://127.0.0.1:9002/d3".to_string(),
             case6_endpoint1.clone(),
         ];
@@ -864,8 +867,8 @@ mod test {
             // Erasure Single Drive
             TestCase {
                 num: 2,
-                server_addr: "localhost:9000",
-                args: vec!["http://localhost/d1"],
+                server_addr: "127.0.0.1:9000",
+                args: vec!["http://127.0.0.1/d1"],
                 expected_err: Some(Error::other("use path style endpoint for single node setup")),
                 ..Default::default()
             },
@@ -885,7 +888,7 @@ mod test {
             },
             TestCase {
                 num: 4,
-                server_addr: "localhost:10000",
+                server_addr: "127.0.0.1:10000",
                 args: vec!["/d1"],
                 expected_endpoints: Some(Endpoints(vec![Endpoint {
                     url: must_file_path("/d1"),
@@ -899,12 +902,12 @@ mod test {
             },
             TestCase {
                 num: 5,
-                server_addr: "localhost:9000",
+                server_addr: "127.0.0.1:9000",
                 args: vec![
                     "https://127.0.0.1:9000/d1",
-                    "https://localhost:9001/d1",
-                    "https://example.com/d1",
-                    "https://example.com/d2",
+                    "https://127.0.0.1:9001/d1",
+                    "https://192.0.2.1/d1",
+                    "https://192.0.2.1/d2",
                 ],
                 expected_err: Some(Error::other("same path '/d1' can not be served by different port on same address")),
                 ..Default::default()
@@ -952,35 +955,35 @@ mod test {
                 num: 7,
                 server_addr: "0.0.0.0:9000",
                 args: vec![
-                    "http://localhost/d1",
-                    "http://localhost/d2",
-                    "http://localhost/d3",
-                    "http://localhost/d4",
+                    "http://127.0.0.1/d1",
+                    "http://127.0.0.1/d2",
+                    "http://127.0.0.1/d3",
+                    "http://127.0.0.1/d4",
                 ],
                 expected_endpoints: Some(Endpoints(vec![
                     Endpoint {
-                        url: must_url("http://localhost:9000/d1"),
+                        url: must_url("http://127.0.0.1:9000/d1"),
                         is_local: true,
                         pool_idx: 0,
                         set_idx: 0,
                         disk_idx: 0,
                     },
                     Endpoint {
-                        url: must_url("http://localhost:9000/d2"),
+                        url: must_url("http://127.0.0.1:9000/d2"),
                         is_local: true,
                         pool_idx: 0,
                         set_idx: 0,
                         disk_idx: 0,
                     },
                     Endpoint {
-                        url: must_url("http://localhost:9000/d3"),
+                        url: must_url("http://127.0.0.1:9000/d3"),
                         is_local: true,
                         pool_idx: 0,
                         set_idx: 0,
                         disk_idx: 0,
                     },
                     Endpoint {
-                        url: must_url("http://localhost:9000/d4"),
+                        url: must_url("http://127.0.0.1:9000/d4"),
                         is_local: true,
                         pool_idx: 0,
                         set_idx: 0,
@@ -995,8 +998,8 @@ mod test {
                 num: 8,
                 server_addr: "127.0.0.1:10000",
                 args: vec![
-                    "http://localhost/d1",
-                    "http://localhost/d2",
+                    "http://[::1]/d1",
+                    "http://[::1]/d2",
                     "http://127.0.0.1/d3",
                     "http://127.0.0.1/d4",
                 ],
@@ -1034,8 +1037,8 @@ mod test {
                 args: vec![
                     case1_endpoint1.as_str(),
                     case1_endpoint2.as_str(),
-                    "http://example.org/d3",
-                    "http://example.com/d4",
+                    "http://192.0.2.10/d3",
+                    "http://192.0.2.11/d4",
                 ],
                 expected_endpoints: Some(Endpoints(vec![
                     Endpoint {
@@ -1076,8 +1079,8 @@ mod test {
                 args: vec![
                     case2_endpoint1.as_str(),
                     case2_endpoint2.as_str(),
-                    "http://example.org/d3",
-                    "http://example.com/d4",
+                    "http://192.0.2.10/d3",
+                    "http://192.0.2.11/d4",
                 ],
                 expected_endpoints: Some(Endpoints(vec![
                     Endpoint {
@@ -1117,9 +1120,9 @@ mod test {
                 server_addr: "0.0.0.0:80",
                 args: vec![
                     case3_endpoint1.as_str(),
-                    "http://example.org:9000/d2",
-                    "http://example.com/d3",
-                    "http://example.net/d4",
+                    "http://192.0.2.10:9000/d2",
+                    "http://192.0.2.11/d3",
+                    "http://192.0.2.12/d4",
                 ],
                 expected_endpoints: Some(Endpoints(vec![
                     Endpoint {
@@ -1159,9 +1162,9 @@ mod test {
                 server_addr: "0.0.0.0:9000",
                 args: vec![
                     case4_endpoint1.as_str(),
-                    "http://example.org/d2",
-                    "http://example.com/d3",
-                    "http://example.net/d4",
+                    "http://192.0.2.10/d2",
+                    "http://192.0.2.11/d3",
+                    "http://192.0.2.12/d4",
                 ],
                 expected_endpoints: Some(Endpoints(vec![
                     Endpoint {
@@ -1242,8 +1245,8 @@ mod test {
                 num: 16,
                 server_addr: "0.0.0.0:9003",
                 args: vec![
-                    "http://localhost:9000/d1",
-                    "http://localhost:9001/d2",
+                    "http://127.0.0.1:9000/d1",
+                    "http://127.0.0.1:9001/d2",
                     "http://127.0.0.1:9002/d3",
                     case6_endpoint1.as_str(),
                 ],

crates/ecstore/src/erasure_coding/bitrot.rs

@@ -15,6 +15,7 @@
 use bytes::Bytes;
 use pin_project_lite::pin_project;
 use rustfs_utils::HashAlgorithm;
+use std::io::IoSlice;
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use tracing::error;
 use uuid::Uuid;
@@ -155,23 +156,49 @@ where
                 error!("bitrot writer write hash error: hash is empty");
                 return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "hash is empty"));
             }
-            self.inner.write_all(hash.as_ref()).await?;
+            write_all_vectored(&mut self.inner, hash.as_ref(), buf).await?;
+        } else {
+            self.inner.write_all(buf).await?;
         }
 
-        self.inner.write_all(buf).await?;
-
-        self.inner.flush().await?;
-
         let n = buf.len();
 
         Ok(n)
     }
 
     pub async fn shutdown(&mut self) -> std::io::Result<()> {
+        self.inner.flush().await?;
         self.inner.shutdown().await
     }
 }
 
+async fn write_all_vectored<W>(writer: &mut W, hash: &[u8], data: &[u8]) -> std::io::Result<()>
+where
+    W: AsyncWrite + Unpin,
+{
+    let mut hash_offset = 0;
+    let mut data_offset = 0;
+
+    while hash_offset < hash.len() || data_offset < data.len() {
+        let slices = [IoSlice::new(&hash[hash_offset..]), IoSlice::new(&data[data_offset..])];
+        let written = writer.write_vectored(&slices).await?;
+        if written == 0 {
+            return Err(std::io::Error::new(std::io::ErrorKind::WriteZero, "failed to write hash and data"));
+        }
+
+        let hash_remaining = hash.len() - hash_offset;
+        if written < hash_remaining {
+            hash_offset += written;
+            continue;
+        }
+
+        hash_offset = hash.len();
+        data_offset += written - hash_remaining;
+    }
+
+    Ok(())
+}
+
 pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize {
     if algo != HashAlgorithm::HighwayHash256S && algo != HashAlgorithm::HighwayHash256SLegacy {
         return size;
@@ -292,6 +319,33 @@ impl AsyncWrite for CustomWriter {
             }
         }
     }
+
+    fn poll_write_vectored(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        bufs: &[IoSlice<'_>],
+    ) -> std::task::Poll<std::io::Result<usize>> {
+        match self.get_mut() {
+            Self::InlineBuffer(data) => {
+                let total = bufs.iter().map(|buf| buf.len()).sum::<usize>();
+                for buf in bufs {
+                    data.extend_from_slice(buf);
+                }
+                std::task::Poll::Ready(Ok(total))
+            }
+            Self::Other(writer) => {
+                let pinned_writer = std::pin::Pin::new(writer.as_mut());
+                pinned_writer.poll_write_vectored(cx, bufs)
+            }
+        }
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        match self {
+            Self::InlineBuffer(_) => true,
+            Self::Other(writer) => writer.is_write_vectored(),
+        }
+    }
 }
 
 /// Wrapper around BitrotWriter that uses our custom writer
@@ -361,7 +415,74 @@ mod tests {
     use super::BitrotReader;
     use super::BitrotWriter;
     use rustfs_utils::HashAlgorithm;
-    use std::io::Cursor;
+    use std::io::{Cursor, IoSlice};
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+    use std::task::{Context, Poll};
+    use tokio::io::AsyncWrite;
+
+    #[derive(Default)]
+    struct VectoredCountingWriter {
+        vectored_writes: Arc<AtomicUsize>,
+        writes: Vec<u8>,
+    }
+
+    impl AsyncWrite for VectoredCountingWriter {
+        fn poll_write(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            Poll::Ready(Err(std::io::Error::other("poll_write should not be used")))
+        }
+
+        fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_write_vectored(
+            mut self: std::pin::Pin<&mut Self>,
+            _cx: &mut Context<'_>,
+            bufs: &[IoSlice<'_>],
+        ) -> Poll<std::io::Result<usize>> {
+            self.vectored_writes.fetch_add(1, Ordering::SeqCst);
+            let total = bufs.iter().map(|buf| buf.len()).sum::<usize>();
+            for buf in bufs {
+                self.writes.extend_from_slice(buf);
+            }
+            Poll::Ready(Ok(total))
+        }
+
+        fn is_write_vectored(&self) -> bool {
+            true
+        }
+    }
+
+    #[derive(Default)]
+    struct CountingWriter {
+        flushes: Arc<AtomicUsize>,
+        shutdowns: Arc<AtomicUsize>,
+        writes: Vec<u8>,
+    }
+
+    impl AsyncWrite for CountingWriter {
+        fn poll_write(mut self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            self.writes.extend_from_slice(buf);
+            Poll::Ready(Ok(buf.len()))
+        }
+
+        fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            self.flushes.fetch_add(1, Ordering::SeqCst);
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            self.shutdowns.fetch_add(1, Ordering::SeqCst);
+            Poll::Ready(Ok(()))
+        }
+    }
 
     #[tokio::test]
     async fn test_bitrot_read_write_ok() {
@@ -471,4 +592,41 @@ mod tests {
         assert_eq!(n, data_size);
         assert_eq!(data, &out[..]);
     }
+
+    #[tokio::test]
+    async fn test_bitrot_writer_flushes_once_on_shutdown() {
+        let flushes = Arc::new(AtomicUsize::new(0));
+        let shutdowns = Arc::new(AtomicUsize::new(0));
+        let writer = CountingWriter {
+            flushes: flushes.clone(),
+            shutdowns: shutdowns.clone(),
+            writes: Vec::new(),
+        };
+        let mut bitrot_writer = BitrotWriter::new(writer, 8, HashAlgorithm::None);
+
+        bitrot_writer.write(b"12345678").await.unwrap();
+        bitrot_writer.write(b"abc").await.unwrap();
+
+        assert_eq!(flushes.load(Ordering::SeqCst), 0);
+        assert_eq!(shutdowns.load(Ordering::SeqCst), 0);
+
+        bitrot_writer.shutdown().await.unwrap();
+
+        assert_eq!(flushes.load(Ordering::SeqCst), 1);
+        assert_eq!(shutdowns.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn test_bitrot_writer_uses_vectored_write_for_hash_and_data() {
+        let vectored_writes = Arc::new(AtomicUsize::new(0));
+        let writer = VectoredCountingWriter {
+            vectored_writes: vectored_writes.clone(),
+            writes: Vec::new(),
+        };
+        let mut bitrot_writer = BitrotWriter::new(writer, 8, HashAlgorithm::HighwayHash256);
+
+        bitrot_writer.write(b"payload").await.unwrap();
+
+        assert!(vectored_writes.load(Ordering::SeqCst) > 0);
+    }
 }

crates/ecstore/src/erasure_coding/encode.rs

@@ -112,11 +112,66 @@ impl<'a> MultiWriter<'a> {
         )))
     }
 
-    pub async fn _shutdown(&mut self) -> std::io::Result<()> {
-        for writer in self.writers.iter_mut().flatten() {
-            writer.shutdown().await?;
+    async fn shutdown_writer(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>) {
+        match writer_opt {
+            Some(writer) => match writer.shutdown().await {
+                Ok(()) => {
+                    *err = None;
+                }
+                Err(e) => {
+                    *err = Some(Error::from(e));
+                    *writer_opt = None;
+                }
+            },
+            None => {
+                *err = Some(Error::DiskNotFound);
+            }
         }
-        Ok(())
+    }
+
+    pub async fn shutdown(&mut self) -> std::io::Result<()> {
+        {
+            let mut futures = FuturesUnordered::new();
+            for (writer_opt, err) in self.writers.iter_mut().zip(self.errs.iter_mut()) {
+                if err.is_some() {
+                    continue;
+                }
+                futures.push(Self::shutdown_writer(writer_opt, err));
+            }
+            while let Some(()) = futures.next().await {}
+        }
+
+        let nil_count = self.errs.iter().filter(|&e| e.is_none()).count();
+        if nil_count >= self.write_quorum {
+            return Ok(());
+        }
+
+        if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
+            error!(
+                "reduce_write_quorum_errs during shutdown: {:?}, offline-disks={}/{}, errs={:?}",
+                write_err,
+                count_errs(&self.errs, &Error::DiskNotFound),
+                self.writers.len(),
+                self.errs
+            );
+            return Err(std::io::Error::other(format!(
+                "Failed to shutdown writers: {} (offline-disks={}/{})",
+                write_err,
+                count_errs(&self.errs, &Error::DiskNotFound),
+                self.writers.len()
+            )));
+        }
+
+        Err(std::io::Error::other(format!(
+            "Failed to shutdown writers: (offline-disks={}/{}): {}",
+            count_errs(&self.errs, &Error::DiskNotFound),
+            self.writers.len(),
+            self.errs
+                .iter()
+                .map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
+                .collect::<Vec<_>>()
+                .join(", ")
+        )))
     }
 }
 
@@ -176,7 +231,69 @@ impl Erasure {
         }
 
         let (reader, total) = task.await??;
-        // writers.shutdown().await?;
+        writers.shutdown().await?;
         Ok((reader, total))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::erasure_coding::{BitrotWriterWrapper, CustomWriter};
+    use rustfs_utils::HashAlgorithm;
+    use std::pin::Pin;
+    use std::sync::{Arc, Mutex};
+    use std::task::{Context, Poll};
+    use tokio::io::AsyncWrite;
+
+    #[derive(Clone, Default)]
+    struct DeferredCommitWriter {
+        buffered: Vec<u8>,
+        committed: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl DeferredCommitWriter {
+        fn new(committed: Arc<Mutex<Vec<u8>>>) -> Self {
+            Self {
+                buffered: Vec::new(),
+                committed,
+            }
+        }
+    }
+
+    impl AsyncWrite for DeferredCommitWriter {
+        fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            self.buffered.extend_from_slice(buf);
+            Poll::Ready(Ok(buf.len()))
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            let buffered = std::mem::take(&mut self.buffered);
+            let mut committed = self.committed.lock().unwrap();
+            committed.extend_from_slice(&buffered);
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[tokio::test]
+    async fn encode_shutdowns_writers_after_small_shards() {
+        let committed = Arc::new(Mutex::new(Vec::new()));
+        let writer = DeferredCommitWriter::new(committed.clone());
+        let mut writers = vec![Some(BitrotWriterWrapper::new(
+            CustomWriter::new_tokio_writer(writer),
+            16,
+            HashAlgorithm::HighwayHash256S,
+        ))];
+
+        let erasure = Arc::new(Erasure::new(1, 0, 16));
+        let reader = tokio::io::BufReader::new(std::io::Cursor::new(b"small payload".to_vec()));
+        let (_reader, written) = erasure.encode(reader, &mut writers, 1).await.unwrap();
+
+        assert_eq!(written, b"small payload".len());
+        assert!(!committed.lock().unwrap().is_empty());
+    }
+}

crates/ecstore/src/file_cache.rs

@@ -1,332 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! High-performance file content and metadata caching using moka
-//!
-//! This module provides optimized caching for file operations to reduce
-//! redundant I/O and improve overall system performance.
-
-use super::disk::error::{Error, Result};
-use bytes::Bytes;
-use moka::future::Cache;
-use rustfs_filemeta::FileMeta;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use std::time::Duration;
-
-pub struct OptimizedFileCache {
-    // Use moka as high-performance async cache
-    metadata_cache: Cache<PathBuf, Arc<FileMeta>>,
-    file_content_cache: Cache<PathBuf, Bytes>,
-    // Performance monitoring
-    cache_hits: std::sync::atomic::AtomicU64,
-    cache_misses: std::sync::atomic::AtomicU64,
-}
-
-impl OptimizedFileCache {
-    pub fn new() -> Self {
-        Self {
-            metadata_cache: Cache::builder()
-                .max_capacity(2048)
-                .time_to_live(Duration::from_secs(300)) // 5 minutes TTL
-                .time_to_idle(Duration::from_secs(60)) // 1 minute idle
-                .build(),
-
-            file_content_cache: Cache::builder()
-                .max_capacity(512) // Smaller file content cache
-                .time_to_live(Duration::from_secs(120))
-                .weigher(|_key: &PathBuf, value: &Bytes| value.len() as u32)
-                .build(),
-
-            cache_hits: std::sync::atomic::AtomicU64::new(0),
-            cache_misses: std::sync::atomic::AtomicU64::new(0),
-        }
-    }
-
-    pub async fn get_metadata(&self, path: PathBuf) -> Result<Arc<FileMeta>> {
-        if let Some(cached) = self.metadata_cache.get(&path).await {
-            self.cache_hits.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-            return Ok(cached);
-        }
-
-        self.cache_misses.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-
-        // Cache miss, read file
-        let data = tokio::fs::read(&path)
-            .await
-            .map_err(|e| Error::other(format!("Read metadata failed: {e}")))?;
-
-        let mut meta = FileMeta::default();
-        meta.unmarshal_msg(&data)?;
-
-        let arc_meta = Arc::new(meta);
-        self.metadata_cache.insert(path, arc_meta.clone()).await;
-
-        Ok(arc_meta)
-    }
-
-    pub async fn get_file_content(&self, path: PathBuf) -> Result<Bytes> {
-        if let Some(cached) = self.file_content_cache.get(&path).await {
-            self.cache_hits.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-            return Ok(cached);
-        }
-
-        self.cache_misses.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-
-        let data = tokio::fs::read(&path)
-            .await
-            .map_err(|e| Error::other(format!("Read file failed: {e}")))?;
-
-        let bytes = Bytes::from(data);
-        self.file_content_cache.insert(path, bytes.clone()).await;
-
-        Ok(bytes)
-    }
-
-    // Prefetch related files
-    pub async fn prefetch_related(&self, base_path: &Path, patterns: &[&str]) {
-        let mut prefetch_tasks = Vec::new();
-
-        for pattern in patterns {
-            let path = base_path.join(pattern);
-            if tokio::fs::metadata(&path).await.is_ok() {
-                let cache = self.clone();
-                let path_clone = path.clone();
-                prefetch_tasks.push(async move {
-                    let _ = cache.get_metadata(path_clone).await;
-                });
-            }
-        }
-
-        // Parallel prefetch, don't wait for completion
-        if !prefetch_tasks.is_empty() {
-            tokio::spawn(async move {
-                futures::future::join_all(prefetch_tasks).await;
-            });
-        }
-    }
-
-    // Batch metadata reading with deduplication
-    pub async fn get_metadata_batch(
-        &self,
-        paths: Vec<PathBuf>,
-    ) -> Vec<std::result::Result<Arc<FileMeta>, rustfs_filemeta::Error>> {
-        let mut results = Vec::with_capacity(paths.len());
-        let mut cache_futures = Vec::new();
-
-        // First, attempt to get from cache
-        for (i, path) in paths.iter().enumerate() {
-            if let Some(cached) = self.metadata_cache.get(path).await {
-                results.push((i, Ok(cached)));
-                self.cache_hits.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-            } else {
-                cache_futures.push((i, path.clone()));
-            }
-        }
-
-        // For cache misses, read from filesystem
-        if !cache_futures.is_empty() {
-            let mut fs_results = Vec::new();
-
-            for (i, path) in cache_futures {
-                self.cache_misses.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-
-                match tokio::fs::read(&path).await {
-                    Ok(data) => {
-                        let mut meta = FileMeta::default();
-                        match meta.unmarshal_msg(&data) {
-                            Ok(_) => {
-                                let arc_meta = Arc::new(meta);
-                                self.metadata_cache.insert(path, arc_meta.clone()).await;
-                                fs_results.push((i, Ok(arc_meta)));
-                            }
-                            Err(e) => {
-                                fs_results.push((i, Err(e)));
-                            }
-                        }
-                    }
-                    Err(_e) => {
-                        fs_results.push((i, Err(rustfs_filemeta::Error::Unexpected)));
-                    }
-                }
-            }
-
-            results.extend(fs_results);
-        }
-
-        // Sort results back to original order
-        results.sort_by_key(|(i, _)| *i);
-        results.into_iter().map(|(_, result)| result).collect()
-    }
-
-    // Invalidate cache entries for a path
-    pub async fn invalidate(&self, path: &Path) {
-        self.metadata_cache.remove(path).await;
-        self.file_content_cache.remove(path).await;
-    }
-
-    // Get cache statistics
-    pub fn get_stats(&self) -> FileCacheStats {
-        let hits = self.cache_hits.load(std::sync::atomic::Ordering::Relaxed);
-        let misses = self.cache_misses.load(std::sync::atomic::Ordering::Relaxed);
-        let hit_rate = if hits + misses > 0 {
-            (hits as f64 / (hits + misses) as f64) * 100.0
-        } else {
-            0.0
-        };
-
-        FileCacheStats {
-            metadata_cache_size: self.metadata_cache.entry_count(),
-            content_cache_size: self.file_content_cache.entry_count(),
-            cache_hits: hits,
-            cache_misses: misses,
-            hit_rate,
-            total_weight: 0, // Simplified for compatibility
-        }
-    }
-
-    // Clear all caches
-    pub async fn clear(&self) {
-        self.metadata_cache.invalidate_all();
-        self.file_content_cache.invalidate_all();
-
-        // Wait for invalidation to complete
-        self.metadata_cache.run_pending_tasks().await;
-        self.file_content_cache.run_pending_tasks().await;
-    }
-}
-
-impl Clone for OptimizedFileCache {
-    fn clone(&self) -> Self {
-        Self {
-            metadata_cache: self.metadata_cache.clone(),
-            file_content_cache: self.file_content_cache.clone(),
-            cache_hits: std::sync::atomic::AtomicU64::new(self.cache_hits.load(std::sync::atomic::Ordering::Relaxed)),
-            cache_misses: std::sync::atomic::AtomicU64::new(self.cache_misses.load(std::sync::atomic::Ordering::Relaxed)),
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct FileCacheStats {
-    pub metadata_cache_size: u64,
-    pub content_cache_size: u64,
-    pub cache_hits: u64,
-    pub cache_misses: u64,
-    pub hit_rate: f64,
-    pub total_weight: u64,
-}
-
-impl Default for OptimizedFileCache {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-// Global cache instance
-use std::sync::OnceLock;
-
-static GLOBAL_FILE_CACHE: OnceLock<OptimizedFileCache> = OnceLock::new();
-
-pub fn get_global_file_cache() -> &'static OptimizedFileCache {
-    GLOBAL_FILE_CACHE.get_or_init(OptimizedFileCache::new)
-}
-
-// Utility functions for common operations
-pub async fn read_metadata_cached(path: PathBuf) -> Result<Arc<FileMeta>> {
-    get_global_file_cache().get_metadata(path).await
-}
-
-pub async fn read_file_content_cached(path: PathBuf) -> Result<Bytes> {
-    get_global_file_cache().get_file_content(path).await
-}
-
-pub async fn prefetch_metadata_patterns(base_path: &Path, patterns: &[&str]) {
-    get_global_file_cache().prefetch_related(base_path, patterns).await;
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::io::Write;
-    use tempfile::tempdir;
-
-    #[tokio::test]
-    async fn test_file_cache_basic() {
-        let cache = OptimizedFileCache::new();
-
-        // Create a temporary file
-        let dir = tempdir().unwrap();
-        let file_path = dir.path().join("test.txt");
-        let mut file = std::fs::File::create(&file_path).unwrap();
-        writeln!(file, "test content").unwrap();
-        drop(file);
-
-        // First read should be cache miss
-        let content1 = cache.get_file_content(file_path.clone()).await.unwrap();
-        assert_eq!(content1, Bytes::from("test content\n"));
-
-        // Second read should be cache hit
-        let content2 = cache.get_file_content(file_path.clone()).await.unwrap();
-        assert_eq!(content2, content1);
-
-        let stats = cache.get_stats();
-        assert!(stats.cache_hits > 0);
-        assert!(stats.cache_misses > 0);
-    }
-
-    #[tokio::test]
-    async fn test_metadata_batch_read() {
-        let cache = OptimizedFileCache::new();
-
-        // Create test files
-        let dir = tempdir().unwrap();
-        let mut paths = Vec::new();
-
-        for i in 0..5 {
-            let file_path = dir.path().join(format!("test_{i}.txt"));
-            let mut file = std::fs::File::create(&file_path).unwrap();
-            writeln!(file, "content {i}").unwrap();
-            paths.push(file_path);
-        }
-
-        // Note: This test would need actual FileMeta files to work properly
-        // For now, we just test that the function runs without errors
-        let results = cache.get_metadata_batch(paths).await;
-        assert_eq!(results.len(), 5);
-    }
-
-    #[tokio::test]
-    async fn test_cache_invalidation() {
-        let cache = OptimizedFileCache::new();
-
-        let dir = tempdir().unwrap();
-        let file_path = dir.path().join("test.txt");
-        let mut file = std::fs::File::create(&file_path).unwrap();
-        writeln!(file, "test content").unwrap();
-        drop(file);
-
-        // Read file to populate cache
-        let _ = cache.get_file_content(file_path.clone()).await.unwrap();
-
-        // Invalidate cache
-        cache.invalidate(&file_path).await;
-
-        // Next read should be cache miss again
-        let _ = cache.get_file_content(file_path.clone()).await.unwrap();
-
-        let stats = cache.get_stats();
-        assert!(stats.cache_misses >= 2);
-    }
-}

crates/ecstore/src/global.rs

@@ -46,6 +46,7 @@ lazy_static! {
     pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
     pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
     pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
+    pub static ref GLOBAL_LOCAL_DISK_ID_MAP: Arc<RwLock<HashMap<Uuid, String>>> = Arc::new(RwLock::new(HashMap::new()));
     pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
     pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new();
     pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
@@ -153,6 +154,10 @@ pub fn get_global_endpoints_opt() -> Option<EndpointServerPools> {
     GLOBAL_Endpoints.get().cloned()
 }
 
+pub async fn is_first_cluster_node_local() -> bool {
+    get_global_endpoints().first_local()
+}
+
 pub fn get_global_tier_config_mgr() -> Arc<RwLock<TierConfigMgr>> {
     GLOBAL_TierConfigMgr.clone()
 }

crates/ecstore/src/lib.rs

@@ -28,7 +28,6 @@ pub mod disks_layout;
 pub mod endpoints;
 pub mod erasure_coding;
 pub mod error;
-pub mod file_cache;
 pub mod global;
 pub mod metrics_realtime;
 pub mod notification_sys;

crates/ecstore/src/metrics_realtime.rs

@@ -14,8 +14,11 @@
 
 use crate::{admin_server_info::get_local_server_property, new_object_layer_fn, store_api::StorageAPI};
 use chrono::Utc;
-use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, metrics::global_metrics};
-use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, RealtimeMetrics};
+use rustfs_common::{
+    GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, internode_metrics::global_internode_metrics,
+    metrics::global_metrics,
+};
+use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics};
 use rustfs_utils::os::get_drive_stats;
 use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
@@ -120,13 +123,50 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
 
     // if types.contains(&MetricType::SITE_RESYNC) {}
 
-    // if types.contains(&MetricType::NET) {}
+    if types.contains(&MetricType::NET) {
+        let snapshot = global_internode_metrics().snapshot();
+        real_time_metrics.aggregated.net = Some(NetMetrics {
+            collected_at: Utc::now(),
+            interface_name: "internode".to_string(),
+            net_stats: NetDevLine {
+                name: "internode".to_string(),
+                rx_bytes: snapshot.recv_bytes_total,
+                tx_bytes: snapshot.sent_bytes_total,
+                ..Default::default()
+            },
+        });
+    }
 
     // if types.contains(&MetricType::MEM) {}
 
     // if types.contains(&MetricType::CPU) {}
 
-    // if types.contains(&MetricType::RPC) {}
+    if types.contains(&MetricType::RPC) {
+        let collected_at = Utc::now();
+        let snapshot = global_internode_metrics().snapshot();
+        let last_connect_time =
+            chrono::DateTime::<Utc>::from_timestamp_millis(snapshot.last_dial_unix_millis as i64).unwrap_or(collected_at);
+
+        real_time_metrics.aggregated.rpc = Some(RPCMetrics {
+            collected_at,
+            connected: i32::from(snapshot.last_dial_unix_millis > 0),
+            reconnect_count: snapshot.dial_errors_total.min(i32::MAX as u64) as i32,
+            disconnected: 0,
+            outgoing_streams: 0,
+            incoming_streams: 0,
+            outgoing_bytes: snapshot.sent_bytes_total.min(i64::MAX as u64) as i64,
+            incoming_bytes: snapshot.recv_bytes_total.min(i64::MAX as u64) as i64,
+            outgoing_messages: snapshot.outgoing_requests_total.min(i64::MAX as u64) as i64,
+            incoming_messages: snapshot.incoming_requests_total.min(i64::MAX as u64) as i64,
+            out_queue: 0,
+            last_pong_time: collected_at,
+            last_ping_ms: snapshot.dial_avg_time_nanos as f64 / 1_000_000.0,
+            max_ping_dur_ms: snapshot.dial_avg_time_nanos as f64 / 1_000_000.0,
+            last_connect_time,
+            by_destination: None,
+            by_caller: None,
+        });
+    }
 
     real_time_metrics
         .by_host
@@ -211,7 +251,9 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
 
 #[cfg(test)]
 mod test {
-    use super::MetricType;
+    use super::*;
+    use rustfs_common::internode_metrics::global_internode_metrics;
+    use std::time::Duration;
 
     #[test]
     fn tes_types() {
@@ -229,4 +271,30 @@ mod test {
         let disk = MetricType::new(1 << 1);
         assert!(disk.contains(&MetricType::DISK));
     }
+
+    #[tokio::test]
+    async fn collect_local_metrics_reports_internode_net_and_rpc() {
+        let metrics = global_internode_metrics();
+        metrics.reset_for_test();
+        metrics.record_sent_bytes(128);
+        metrics.record_recv_bytes(64);
+        metrics.record_outgoing_request();
+        metrics.record_incoming_request();
+        metrics.record_dial_result(Duration::from_millis(4), true);
+
+        let realtime = collect_local_metrics(MetricType::NET, &CollectMetricsOpts::default()).await;
+        let net = realtime.aggregated.net.expect("net metrics");
+        assert_eq!(net.net_stats.tx_bytes, 128);
+        assert_eq!(net.net_stats.rx_bytes, 64);
+
+        let realtime = collect_local_metrics(MetricType::RPC, &CollectMetricsOpts::default()).await;
+        let rpc = realtime.aggregated.rpc.expect("rpc metrics");
+        assert_eq!(rpc.outgoing_bytes, 128);
+        assert_eq!(rpc.incoming_bytes, 64);
+        assert_eq!(rpc.outgoing_messages, 1);
+        assert_eq!(rpc.incoming_messages, 1);
+        assert!(rpc.last_ping_ms > 0.0);
+
+        metrics.reset_for_test();
+    }
 }

crates/ecstore/src/pools.rs

@@ -46,7 +46,9 @@ use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
 use bytes::Bytes;
 use futures::future::BoxFuture;
 use http::HeaderMap;
-use rmp_serde::{Deserializer, Serializer};
+#[cfg(test)]
+use rmp_serde::Deserializer;
+use rmp_serde::Serializer;
 use rustfs_common::defer;
 use rustfs_common::heal_channel::HealOpts;
 use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
@@ -235,6 +237,22 @@ impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo {
 }
 
 impl PoolMeta {
+    fn decode_pool_meta_payload(payload: &[u8]) -> Result<Self> {
+        match rmp_serde::from_slice::<PersistedPoolMeta>(payload) {
+            Ok(meta) => Ok(meta.into()),
+            Err(persisted_err) => {
+                let mut legacy: PoolMeta = rmp_serde::from_slice(payload).map_err(|legacy_err| {
+                    Error::other(format!(
+                        "PoolMeta decode failed for both persisted and legacy formats: persisted={persisted_err}; legacy={legacy_err}"
+                    ))
+                })?;
+                // Runtime-only flag must not be restored from on-disk payload.
+                legacy.dont_save = false;
+                Ok(legacy)
+            }
+        }
+    }
+
     pub fn new(pools: &[Arc<Sets>], prev_meta: &PoolMeta) -> Self {
         let mut new_meta = Self {
             version: POOL_META_VERSION,
@@ -302,9 +320,7 @@ impl PoolMeta {
             return Err(Error::other(format!("PoolMeta: unknown version: {version}")));
         }
 
-        let mut buf = Deserializer::new(Cursor::new(&data[4..]));
-        let meta: PersistedPoolMeta = Deserialize::deserialize(&mut buf)?;
-        *self = meta.into();
+        *self = Self::decode_pool_meta_payload(&data[4..])?;
 
         if self.version != POOL_META_VERSION {
             return Err(Error::other(format!("unexpected PoolMeta version: {}", self.version)));
@@ -1959,6 +1975,63 @@ mod tests {
         assert_eq!(restored_decommission.bytes_done, 1024);
         assert_eq!(restored_decommission.bytes_failed, 128);
     }
+
+    #[test]
+    fn pool_meta_decode_supports_legacy_payload() {
+        let start_time = OffsetDateTime::now_utc();
+        let legacy_meta = PoolMeta {
+            version: POOL_META_VERSION,
+            pools: vec![PoolStatus {
+                id: 3,
+                cmd_line: "/legacy/pool".to_string(),
+                last_update: start_time,
+                decommission: Some(PoolDecommissionInfo {
+                    start_time: Some(start_time),
+                    items_decommissioned: 9,
+                    items_decommission_failed: 2,
+                    bytes_done: 2048,
+                    bytes_failed: 256,
+                    queued_buckets: vec!["not-persisted".to_string()],
+                    decommissioned_buckets: vec!["not-persisted".to_string()],
+                    bucket: "not-persisted".to_string(),
+                    prefix: "not-persisted".to_string(),
+                    object: "not-persisted".to_string(),
+                    ..Default::default()
+                }),
+            }],
+            dont_save: true,
+        };
+
+        let mut legacy_payload = Vec::new();
+        legacy_meta
+            .serialize(&mut Serializer::new(&mut legacy_payload))
+            .expect("legacy payload should serialize");
+
+        // New persisted schema has fewer top-level fields and should not decode this legacy struct payload.
+        let persisted_decode: std::result::Result<PersistedPoolMeta, _> = rmp_serde::from_slice(&legacy_payload);
+        assert!(persisted_decode.is_err());
+
+        let decoded = PoolMeta::decode_pool_meta_payload(&legacy_payload).expect("legacy payload should decode");
+        assert_eq!(decoded.version, POOL_META_VERSION);
+        assert!(!decoded.dont_save, "runtime-only flag should reset on load");
+        assert_eq!(decoded.pools.len(), 1);
+        assert_eq!(decoded.pools[0].id, 3);
+        assert_eq!(decoded.pools[0].cmd_line, "/legacy/pool");
+        assert_eq!(decoded.pools[0].last_update, start_time);
+
+        let decommission = decoded.pools[0].decommission.as_ref().expect("decommission should decode");
+        assert_eq!(decommission.start_time, Some(start_time));
+        assert_eq!(decommission.items_decommissioned, 9);
+        assert_eq!(decommission.items_decommission_failed, 2);
+        assert_eq!(decommission.bytes_done, 2048);
+        assert_eq!(decommission.bytes_failed, 256);
+        // These fields were skipped in legacy payload and should be defaulted.
+        assert!(decommission.queued_buckets.is_empty());
+        assert!(decommission.decommissioned_buckets.is_empty());
+        assert!(decommission.bucket.is_empty());
+        assert!(decommission.prefix.is_empty());
+        assert!(decommission.object.is_empty());
+    }
 }
 
 // impl Fn(MetaCacheEntry) -> impl Future<Output = Result<(), Error>>

crates/ecstore/src/rpc/remote_disk.rs

@@ -23,6 +23,7 @@ use crate::disk::{
 };
 use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
 use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::set_disk::DEFAULT_READ_BUFFER_SIZE;
 use crate::{
     disk::error::{Error, Result},
     rpc::build_auth_headers,
@@ -40,7 +41,9 @@ use rustfs_protos::proto_gen::node_service::{
     node_service_client::NodeServiceClient,
 };
 use rustfs_rio::{HttpReader, HttpWriter};
+use serde::{Serialize, de::DeserializeOwned};
 use std::{
+    io::Cursor,
     path::PathBuf,
     sync::{
         Arc,
@@ -49,12 +52,36 @@ use std::{
     time::Duration,
 };
 use tokio::time;
-use tokio::{io::AsyncWrite, net::TcpStream, time::timeout};
+use tokio::{
+    io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt},
+    net::TcpStream,
+    time::timeout,
+};
 use tokio_util::sync::CancellationToken;
 use tonic::{Request, service::interceptor::InterceptedService, transport::Channel};
 use tracing::{debug, info, warn};
 use uuid::Uuid;
 
+async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
+where
+    R: AsyncRead + Unpin,
+    W: AsyncWrite + Unpin,
+{
+    let mut copied = 0_u64;
+    let mut buffer = vec![0_u8; buffer_size];
+
+    loop {
+        let bytes_read = reader.read(&mut buffer).await?;
+        if bytes_read == 0 {
+            writer.flush().await?;
+            return Ok(copied);
+        }
+
+        writer.write_all(&buffer[..bytes_read]).await?;
+        copied += bytes_read as u64;
+    }
+}
+
 #[derive(Debug)]
 pub struct RemoteDisk {
     pub id: Mutex<Option<Uuid>>,
@@ -259,6 +286,27 @@ impl RemoteDisk {
             .await
             .map_err(|err| Error::other(format!("can not get client, err: {err}")))
     }
+
+    async fn disk_ref(&self) -> String {
+        (*self.id.lock().await)
+            .map(|id| id.to_string())
+            .unwrap_or_else(|| self.endpoint.to_string())
+    }
+}
+
+fn encode_msgpack<T: Serialize>(value: &T) -> Result<Vec<u8>> {
+    let mut serializer = rmp_serde::Serializer::new(Vec::new());
+    value.serialize(&mut serializer)?;
+    Ok(serializer.into_inner())
+}
+
+fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str) -> Result<T> {
+    if !binary.is_empty() {
+        let mut deserializer = rmp_serde::Deserializer::new(Cursor::new(binary));
+        return T::deserialize(&mut deserializer).map_err(Error::from);
+    }
+
+    serde_json::from_str(json).map_err(Error::from)
 }
 
 // TODO: all api need to handle errors
@@ -707,18 +755,21 @@ impl DiskAPI for RemoteDisk {
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
         info!("write_metadata {}/{}", volume, path);
         let file_info = serde_json::to_string(&fi)?;
+        let file_info_bin = encode_msgpack(&fi)?;
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(WriteMetadataRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     file_info: file_info.clone(),
+                    file_info_bin: file_info_bin.clone(),
                 });
 
                 let response = client.write_metadata(request).await?.into_inner();
@@ -735,6 +786,7 @@ impl DiskAPI for RemoteDisk {
     }
 
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
+        let disk = self.disk_ref().await;
         let mut client = self
             .get_client()
             .await
@@ -742,7 +794,7 @@ impl DiskAPI for RemoteDisk {
         let request = Request::new(ReadMetadataRequest {
             volume: volume.to_string(),
             path: path.to_string(),
-            disk: self.endpoint.to_string(),
+            disk,
         });
 
         let response = client.read_metadata(request).await?.into_inner();
@@ -759,19 +811,24 @@ impl DiskAPI for RemoteDisk {
         info!("update_metadata");
         let file_info = serde_json::to_string(&fi)?;
         let opts_str = serde_json::to_string(&opts)?;
+        let file_info_bin = encode_msgpack(&fi)?;
+        let opts_bin = encode_msgpack(opts)?;
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(UpdateMetadataRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     file_info: file_info.clone(),
                     opts: opts_str.clone(),
+                    file_info_bin: file_info_bin.clone(),
+                    opts_bin: opts_bin.clone(),
                 });
 
                 let response = client.update_metadata(request).await?.into_inner();
@@ -798,19 +855,22 @@ impl DiskAPI for RemoteDisk {
     ) -> Result<FileInfo> {
         info!("read_version");
         let opts_str = serde_json::to_string(opts)?;
+        let opts_bin = encode_msgpack(opts)?;
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(ReadVersionRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     version_id: version_id.to_string(),
                     opts: opts_str.clone(),
+                    opts_bin: opts_bin.clone(),
                 });
 
                 let response = client.read_version(request).await?.into_inner();
@@ -819,7 +879,7 @@ impl DiskAPI for RemoteDisk {
                     return Err(response.error.unwrap_or_default().into());
                 }
 
-                let file_info = serde_json::from_str::<FileInfo>(&response.file_info)?;
+                let file_info = decode_msgpack_or_json::<FileInfo>(&response.file_info_bin, &response.file_info)?;
 
                 Ok(file_info)
             },
@@ -834,12 +894,13 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(ReadXlRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     read_data,
@@ -851,7 +912,7 @@ impl DiskAPI for RemoteDisk {
                     return Err(response.error.unwrap_or_default().into());
                 }
 
-                let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
+                let raw_file_info = decode_msgpack_or_json::<RawFileInfo>(&response.raw_file_info_bin, &response.raw_file_info)?;
 
                 Ok(raw_file_info)
             },
@@ -909,13 +970,14 @@ impl DiskAPI for RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
+        let disk = self.disk_ref().await;
 
         let mut client = self
             .get_client()
             .await
             .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
         let request = Request::new(ListDirRequest {
-            disk: self.endpoint.to_string(),
+            disk,
             volume: volume.to_string(),
             dir_path: dir_path.to_string(),
             count,
@@ -937,12 +999,9 @@ impl DiskAPI for RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
+        let disk = self.disk_ref().await;
 
-        let url = format!(
-            "{}/rustfs/rpc/walk_dir?disk={}",
-            self.endpoint.grid_host(),
-            urlencoding::encode(self.endpoint.to_string().as_str()),
-        );
+        let url = format!("{}/rustfs/rpc/walk_dir?disk={}", self.endpoint.grid_host(), urlencoding::encode(&disk),);
 
         let opts = serde_json::to_vec(&opts)?;
 
@@ -952,33 +1011,14 @@ impl DiskAPI for RemoteDisk {
 
         let mut reader = HttpReader::new(url, Method::GET, headers, Some(opts)).await?;
 
-        tokio::io::copy(&mut reader, wr).await?;
+        copy_stream_with_buffer(&mut reader, wr, DEFAULT_READ_BUFFER_SIZE).await?;
 
         Ok(())
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
     async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
-        info!("read_file {}/{}", volume, path);
-
-        if self.health.is_faulty() {
-            return Err(DiskError::FaultyDisk);
-        }
-
-        let url = format!(
-            "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
-            self.endpoint.grid_host(),
-            urlencoding::encode(self.endpoint.to_string().as_str()),
-            urlencoding::encode(volume),
-            urlencoding::encode(path),
-            0,
-            0
-        );
-
-        let mut headers = HeaderMap::new();
-        headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
-        build_auth_headers(&url, &Method::GET, &mut headers);
-        Ok(Box::new(HttpReader::new(url, Method::GET, headers, None).await?))
+        self.read_file_stream(volume, path, 0, 0).await
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
@@ -995,11 +1035,12 @@ impl DiskAPI for RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
+        let disk = self.disk_ref().await;
 
         let url = format!(
             "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
             self.endpoint.grid_host(),
-            urlencoding::encode(self.endpoint.to_string().as_str()),
+            urlencoding::encode(&disk),
             urlencoding::encode(volume),
             urlencoding::encode(path),
             offset,
@@ -1019,11 +1060,12 @@ impl DiskAPI for RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
+        let disk = self.disk_ref().await;
 
         let url = format!(
             "{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
             self.endpoint.grid_host(),
-            urlencoding::encode(self.endpoint.to_string().as_str()),
+            urlencoding::encode(&disk),
             urlencoding::encode(volume),
             urlencoding::encode(path),
             true,
@@ -1049,11 +1091,12 @@ impl DiskAPI for RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
+        let disk = self.disk_ref().await;
 
         let url = format!(
             "{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
             self.endpoint.grid_host(),
-            urlencoding::encode(self.endpoint.to_string().as_str()),
+            urlencoding::encode(&disk),
             urlencoding::encode(volume),
             urlencoding::encode(path),
             false,
@@ -1261,13 +1304,16 @@ impl DiskAPI for RemoteDisk {
         self.execute_with_timeout(
             || async {
                 let read_multiple_req = serde_json::to_string(&req)?;
+                let read_multiple_req_bin = encode_msgpack(&req)?;
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(ReadMultipleRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     read_multiple_req,
+                    read_multiple_req_bin,
                 });
 
                 let response = client.read_multiple(request).await?.into_inner();
@@ -1276,11 +1322,19 @@ impl DiskAPI for RemoteDisk {
                     return Err(response.error.unwrap_or_default().into());
                 }
 
-                let read_multiple_resps = response
-                    .read_multiple_resps
-                    .into_iter()
-                    .filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
-                    .collect();
+                let read_multiple_resps = if !response.read_multiple_resps_bin.is_empty() {
+                    response
+                        .read_multiple_resps_bin
+                        .into_iter()
+                        .filter_map(|buf| decode_msgpack_or_json::<ReadMultipleResp>(&buf, "").ok())
+                        .collect()
+                } else {
+                    response
+                        .read_multiple_resps
+                        .into_iter()
+                        .filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
+                        .collect()
+                };
 
                 Ok(read_multiple_resps)
             },
@@ -1295,12 +1349,13 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(WriteAllRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     data,
@@ -1325,12 +1380,13 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout(
             || async {
+                let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
                 let request = Request::new(ReadAllRequest {
-                    disk: self.endpoint.to_string(),
+                    disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                 });
@@ -1386,6 +1442,7 @@ impl DiskAPI for RemoteDisk {
 mod tests {
     use super::*;
     use std::sync::Once;
+    use tokio::io::duplex;
     use tokio::net::TcpListener;
     use tracing::Level;
     use uuid::Uuid;
@@ -1543,6 +1600,24 @@ mod tests {
         assert!(!remote_disk.is_online().await);
     }
 
+    #[tokio::test]
+    async fn test_copy_stream_with_buffer_copies_full_payload() {
+        let payload = b"walk-dir-stream".repeat(1024);
+        let expected = payload.clone();
+        let (mut write_half, mut read_half) = duplex(128);
+
+        let copy_task = tokio::spawn(async move {
+            let mut cursor = Cursor::new(payload);
+            copy_stream_with_buffer(&mut cursor, &mut write_half, 4 * 1024).await.unwrap();
+        });
+
+        let mut copied = Vec::new();
+        read_half.read_to_end(&mut copied).await.unwrap();
+        copy_task.await.unwrap();
+
+        assert_eq!(copied, expected);
+    }
+
     #[tokio::test]
     async fn test_remote_disk_disk_id() {
         let url = url::Url::parse("http://remote-server:9000").unwrap();
@@ -1579,6 +1654,30 @@ mod tests {
         assert!(cleared_id.is_none());
     }
 
+    #[tokio::test]
+    async fn test_remote_disk_ref_prefers_disk_id() {
+        let url = url::Url::parse("http://remote-server:9000").unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+        let disk_option = DiskOption {
+            cleanup: false,
+            health_check: false,
+        };
+
+        let remote_disk = RemoteDisk::new(&endpoint, &disk_option).await.unwrap();
+        assert_eq!(remote_disk.disk_ref().await, endpoint.to_string());
+
+        let disk_id = Uuid::new_v4();
+        remote_disk.set_disk_id(Some(disk_id)).await.unwrap();
+
+        assert_eq!(remote_disk.disk_ref().await, disk_id.to_string());
+    }
+
     #[tokio::test]
     async fn test_remote_disk_endpoints_with_different_schemes() {
         let test_cases = vec![

crates/ecstore/src/rpc/remote_locker.rs

@@ -52,6 +52,7 @@ impl RemoteClient {
             metadata: LockMetadata::default(),
             priority: LockPriority::Normal,
             deadlock_detection: false,
+            suppress_contention_logs: false,
         }
     }
 

crates/ecstore/src/set_disk.rs

@@ -121,6 +121,21 @@ use uuid::Uuid;
 
 pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
 pub const MAX_PARTS_COUNT: usize = 10000;
+
+/// Get the duplex buffer size from environment variable or use default.
+///
+/// This function reads `RUSTFS_DUPLEX_BUFFER_SIZE` environment variable
+/// to allow runtime configuration of the duplex pipe buffer size.
+/// A larger buffer (e.g., 4MB) helps prevent backpressure-related hangs
+/// when reading large objects (20-26MB) under high concurrency.
+///
+/// Default: 4MB (4 * 1024 * 1024 bytes)
+pub fn get_duplex_buffer_size() -> usize {
+    rustfs_utils::get_env_usize(
+        rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE,
+        rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE,
+    )
+}
 const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1);
 const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
 
@@ -136,7 +151,72 @@ mod write;
 /// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
 /// Defaults to 30 seconds if not set or invalid
 pub fn get_lock_acquire_timeout() -> Duration {
-    Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT,
+        rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT,
+    ))
+}
+
+/// Check if lock optimization is enabled.
+/// When enabled, read locks are released after metadata read instead of
+/// being held for the entire data transfer duration.
+pub fn is_lock_optimization_enabled() -> bool {
+    rustfs_utils::get_env_bool(
+        rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE,
+        rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE,
+    )
+}
+
+/// Check if deadlock detection is enabled.
+/// When enabled, lock operations are recorded for deadlock analysis.
+pub fn is_deadlock_detection_enabled() -> bool {
+    rustfs_utils::get_env_bool(
+        rustfs_config::ENV_OBJECT_DEADLOCK_DETECTION_ENABLE,
+        rustfs_config::DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE,
+    )
+}
+
+/// Record a lock acquisition for deadlock detection.
+/// This records detailed lock information for deadlock analysis.
+/// Returns the lock_id for later release tracking.
+#[inline]
+fn record_lock_acquire(bucket: &str, object: &str, lock_type: &str) -> String {
+    let lock_id = format!("{}:{}", bucket, object);
+
+    if !is_deadlock_detection_enabled() {
+        return lock_id;
+    }
+
+    let request_id = format!("get-{}-{}", bucket, object);
+    let resource = format!("{}/{}", bucket, object);
+
+    // Log with structured fields for analysis
+    debug!(
+        request_id = %request_id,
+        lock_id = %lock_id,
+        lock_type = %lock_type,
+        resource = %resource,
+        "Lock acquired for deadlock tracking"
+    );
+
+    lock_id
+}
+
+/// Record a lock release for deadlock detection.
+#[inline]
+fn record_lock_release(bucket: &str, object: &str, lock_id: &str, lock_type: &str) {
+    if !is_deadlock_detection_enabled() {
+        return;
+    }
+
+    let request_id = format!("get-{}-{}", bucket, object);
+
+    debug!(
+        request_id = %request_id,
+        lock_id = %lock_id,
+        lock_type = %lock_type,
+        "Lock released for deadlock tracking"
+    );
 }
 
 fn build_tiered_decommission_file_info(
@@ -451,20 +531,44 @@ impl ObjectIO for SetDisks {
         h: HeaderMap,
         opts: &ObjectOptions,
     ) -> Result<GetObjectReader> {
+        // Check if lock optimization is enabled
+        // When enabled, read locks are released after metadata read
+        let lock_optimization_enabled = is_lock_optimization_enabled();
+
         // Acquire a shared read-lock early to protect read consistency
         let read_lock_guard = if !opts.no_lock {
-            Some(
-                self.new_ns_lock(bucket, object)
-                    .await?
-                    .get_read_lock(get_lock_acquire_timeout())
-                    .await
-                    .map_err(|e| {
-                        Error::other(format!(
-                            "Failed to acquire read lock: {}",
-                            self.format_lock_error_from_error(bucket, object, "read", &e)
-                        ))
-                    })?,
-            )
+            let acquire_start = Instant::now();
+
+            // Record lock wait for deadlock detection
+            if is_deadlock_detection_enabled() {
+                debug!(
+                    lock_id = format!("{}:{}", bucket, object),
+                    lock_type = "read",
+                    resource = format!("{}/{}", bucket, object),
+                    "Waiting for read lock"
+                );
+            }
+
+            let guard = self
+                .new_ns_lock(bucket, object)
+                .await?
+                .get_read_lock(get_lock_acquire_timeout())
+                .await
+                .map_err(|e| {
+                    Error::other(format!(
+                        "Failed to acquire read lock: {}",
+                        self.format_lock_error_from_error(bucket, object, "read", &e)
+                    ))
+                })?;
+
+            // Record lock acquisition for deadlock detection
+            let _lock_id = record_lock_acquire(bucket, object, "read");
+
+            // Record lock statistics
+            metrics::counter!("rustfs.lock.acquire.total", "type" => "read").increment(1);
+            metrics::histogram!("rustfs.lock.acquire.duration.seconds").record(acquire_start.elapsed().as_secs_f64());
+
+            Some(guard)
         } else {
             None
         };
@@ -512,7 +616,28 @@ impl ObjectIO for SetDisks {
             return Ok(gr);
         }
 
-        let (rd, wd) = tokio::io::duplex(DEFAULT_READ_BUFFER_SIZE);
+        // Lock optimization: release read lock after metadata read if enabled
+        // This reduces lock contention by not holding the lock during data transfer
+        let read_lock_guard = if lock_optimization_enabled {
+            // Record lock release for deadlock detection
+            if read_lock_guard.is_some() {
+                let lock_id = format!("{}:{}", bucket, object);
+                record_lock_release(bucket, object, &lock_id, "read");
+
+                // Record early lock release statistics
+                metrics::counter!("rustfs.lock.release.early.total", "type" => "read").increment(1);
+            }
+            // Explicitly drop the lock guard to release the lock early
+            drop(read_lock_guard);
+            debug!(bucket, object, "Lock optimization: released read lock after metadata read");
+            None
+        } else {
+            read_lock_guard
+        };
+
+        let duplex_buffer_size = get_duplex_buffer_size();
+        let (rd, wd) = tokio::io::duplex(duplex_buffer_size);
+        debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer");
 
         let (reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h)?;
 
@@ -523,9 +648,10 @@ impl ObjectIO for SetDisks {
         let pool_index = self.pool_index;
         let skip_verify = opts.skip_verify_bitrot;
         // Move the read-lock guard into the task so it lives for the duration of the read
+        // Note: when lock optimization is enabled, read_lock_guard is None
         // let _guard_to_hold = _read_lock_guard; // moved into closure below
         tokio::spawn(async move {
-            let _guard = read_lock_guard; // keep guard alive until task ends
+            let _guard = read_lock_guard; // keep guard alive until task ends (None if optimization enabled)
             let mut writer = wd;
             if let Err(e) = Self::get_object_with_fileinfo(
                 &bucket,
@@ -751,7 +877,7 @@ impl ObjectIO for SetDisks {
             pfi.metadata = user_defined.clone();
             if is_inline_buffer {
                 if let Some(writer) = writers[i].take() {
-                    pfi.data = Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default());
+                    pfi.data = Some(writer.into_inline_data().map(Bytes::from).unwrap_or_default());
                 }
 
                 pfi.set_inline_data();
@@ -1093,7 +1219,7 @@ impl ObjectOperations for SetDisks {
         let mut unique_objects: HashSet<String> = HashSet::new();
         for dobj in &objects {
             if unique_objects.insert(dobj.object_name.clone()) {
-                batch = batch.add_write_lock(rustfs_lock::ObjectKey::new(bucket, dobj.object_name.clone()));
+                batch = batch.add_write_lock(ObjectKey::new(bucket, dobj.object_name.clone()));
             }
         }
 
@@ -1817,19 +1943,54 @@ impl ObjectOperations for SetDisks {
         //}
 
         let mut uploaded_parts: Vec<CompletePart> = vec![];
-        let rs: Option<HTTPRangeSpec> = None;
-        let gr = get_transitioned_object_reader(bucket, object, &rs, &HeaderMap::new(), &oi, opts).await;
-        if let Err(err) = gr {
-            return set_restore_header_fn(&mut oi, Some(StorageError::Io(err))).await;
-        }
-        let gr = gr.unwrap();
-
-        for part_info in &oi.parts {
-            let reader = BufReader::new(Cursor::new(vec![] /*gr.stream*/));
+        let parts = oi.parts.clone();
+        let mut part_offset: i64 = 0;
+        for part_info in &parts {
+            let mut part_opts = opts.clone();
+            part_opts.part_number = Some(part_info.number);
+            if part_info.actual_size <= 0 {
+                return set_restore_header_fn(
+                    &mut oi,
+                    Some(Error::other(format!("invalid multipart restore part size {}", part_info.actual_size))),
+                )
+                .await;
+            }
+            let part_end = match part_offset.checked_add(part_info.actual_size - 1) {
+                Some(end) => end,
+                None => {
+                    return set_restore_header_fn(
+                        &mut oi,
+                        Some(Error::other("multipart restore part range overflow".to_string())),
+                    )
+                    .await;
+                }
+            };
+            let rs = Some(HTTPRangeSpec {
+                is_suffix_length: false,
+                start: part_offset,
+                end: part_end,
+            });
+            part_offset = match part_end.checked_add(1) {
+                Some(next) => next,
+                None => {
+                    return set_restore_header_fn(
+                        &mut oi,
+                        Some(Error::other("multipart restore part offset overflow".to_string())),
+                    )
+                    .await;
+                }
+            };
+            let gr = match get_transitioned_object_reader(bucket, object, &rs, &HeaderMap::new(), &oi, &part_opts).await {
+                Ok(reader) => reader,
+                Err(err) => {
+                    return set_restore_header_fn(&mut oi, Some(StorageError::Io(err))).await;
+                }
+            };
+            let reader = BufReader::new(gr.stream);
             let hash_reader = HashReader::new(
                 Box::new(WarpReader::new(reader)),
-                part_info.size as i64,
-                part_info.size as i64,
+                part_info.actual_size,
+                part_info.actual_size,
                 None,
                 None,
                 false,
@@ -1842,7 +2003,7 @@ impl ObjectOperations for SetDisks {
             //if let Err(err) = p_info {
             //    return set_restore_header_fn(&mut oi, err).await;
             //}
-            if p_info.size != part_info.size {
+            if p_info.size as i64 != part_info.actual_size {
                 return set_restore_header_fn(
                     &mut oi,
                     Some(Error::other(ObjectApiError::InvalidObjectState(GenericError {
@@ -2757,8 +2918,7 @@ impl MultipartOperations for SetDisks {
         // Build a lookup map for O(1) part resolution instead of O(n) find() in the loop
         // This optimizes from O(n^2) to O(n) when processing many parts
         use std::collections::HashMap;
-        let part_lookup: HashMap<usize, &rustfs_filemeta::ObjectPartInfo> =
-            curr_fi.parts.iter().map(|part| (part.number, part)).collect();
+        let part_lookup: HashMap<usize, &ObjectPartInfo> = curr_fi.parts.iter().map(|part| (part.number, part)).collect();
 
         for (i, p) in uploaded_parts.iter().enumerate() {
             let Some(ext_part) = part_lookup.get(&p.part_num) else {
@@ -3445,12 +3605,11 @@ async fn disks_with_all_parts(
         if (meta.data.is_some() || meta.size == 0) && !meta.parts.is_empty() {
             if let Some(data) = &meta.data {
                 let checksum_info = meta.erasure.get_checksum_info(meta.parts[0].number);
-                let checksum_algo =
-                    if meta.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
-                        rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
-                    } else {
-                        checksum_info.algorithm
-                    };
+                let checksum_algo = if meta.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
+                    HashAlgorithm::HighwayHash256SLegacy
+                } else {
+                    checksum_info.algorithm
+                };
                 let data_len = data.len();
                 let verify_err = bitrot_verify(
                     Box::new(Cursor::new(data.clone())),

crates/ecstore/src/store.rs

@@ -27,7 +27,7 @@ use crate::bucket::utils::check_new_multipart_args;
 use crate::bucket::utils::check_object_args;
 use crate::bucket::utils::check_put_object_args;
 use crate::bucket::utils::check_put_object_part_args;
-use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
+use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict};
 use crate::config::GLOBAL_STORAGE_CLASS;
 use crate::config::storageclass;
 use crate::disk::endpoint::{Endpoint, EndpointType};
@@ -142,8 +142,8 @@ mod rebalance;
 
 use peer::init_local_peer;
 pub use peer::{
-    all_local_disk, all_local_disk_path, find_local_disk, get_disk_infos, get_disk_via_endpoint, has_space_for, init_local_disks,
-    init_lock_clients,
+    all_local_disk, all_local_disk_path, find_local_disk, find_local_disk_by_ref, get_disk_infos, get_disk_via_endpoint,
+    has_space_for, init_local_disks, init_lock_clients,
 };
 
 #[derive(Debug)]

crates/ecstore/src/store/bucket.rs

@@ -13,6 +13,11 @@
 // limitations under the License.
 
 use super::*;
+use crate::bucket::utils::is_meta_bucketname;
+
+fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
+    created != OffsetDateTime::UNIX_EPOCH
+}
 
 impl ECStore {
     #[instrument(skip(self))]
@@ -68,7 +73,9 @@ impl ECStore {
         let mut info = self.peer_sys.get_bucket_info(bucket, opts).await?;
 
         if let Ok(sys) = metadata_sys::get(bucket).await {
-            info.created = Some(sys.created);
+            if should_override_created_from_metadata(sys.created) {
+                info.created = Some(sys.created);
+            }
             info.versioning = sys.versioning();
             info.object_locking = sys.object_locking();
         }
@@ -84,7 +91,9 @@ impl ECStore {
 
         if !opts.no_metadata {
             for bucket in buckets.iter_mut() {
-                if let Ok(created) = metadata_sys::created_at(&bucket.name).await {
+                if let Ok(created) = metadata_sys::created_at(&bucket.name).await
+                    && should_override_created_from_metadata(created)
+                {
                     bucket.created = Some(created);
                 }
             }
@@ -148,3 +157,20 @@ impl ECStore {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::should_override_created_from_metadata;
+    use time::OffsetDateTime;
+
+    #[test]
+    fn should_not_override_when_metadata_created_is_unix_epoch() {
+        assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH));
+    }
+
+    #[test]
+    fn should_override_when_metadata_created_is_valid_time() {
+        let created = OffsetDateTime::from_unix_timestamp(1704067200).expect("valid timestamp");
+        assert!(should_override_created_from_metadata(created));
+    }
+}

crates/ecstore/src/store/init.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use crate::global::is_first_cluster_node_local;
 
 impl ECStore {
     #[allow(clippy::new_ret_no_self)]
@@ -203,6 +204,7 @@ impl ECStore {
         let mut meta = PoolMeta::default();
         meta.load(self.pools[0].clone(), self.pools.clone()).await?;
         let update = meta.validate(self.pools.clone())?;
+        let should_persist_pool_meta = is_first_cluster_node_local().await;
 
         if !update {
             {
@@ -211,7 +213,9 @@ impl ECStore {
             }
         } else {
             let new_meta = PoolMeta::new(&self.pools, &meta);
-            new_meta.save(self.pools.clone()).await?;
+            if should_persist_pool_meta {
+                new_meta.save(self.pools.clone()).await?;
+            }
             {
                 let mut pool_meta = self.pool_meta.write().await;
                 *pool_meta = new_meta;

crates/ecstore/src/store/peer.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 
 pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
     let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await;
@@ -24,6 +25,19 @@ pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
     }
 }
 
+pub async fn find_local_disk_by_ref(disk_ref: &str) -> Option<DiskStore> {
+    if let Some(disk) = find_local_disk(&disk_ref.to_string()).await {
+        return Some(disk);
+    }
+
+    let Ok(disk_id) = Uuid::parse_str(disk_ref) else {
+        return None;
+    };
+
+    let disk_path = GLOBAL_LOCAL_DISK_ID_MAP.read().await.get(&disk_id).cloned()?;
+    find_local_disk(&disk_path).await
+}
+
 pub async fn get_disk_via_endpoint(endpoint: &Endpoint) -> Option<DiskStore> {
     let global_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.read().await;
     if global_set_drives.is_empty() {

crates/ecstore/src/store/rebalance.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use crate::bucket::utils::is_meta_bucketname;
 
 impl ECStore {
     #[instrument(level = "debug", skip(self))]

crates/ecstore/src/tier/tier.rs

@@ -49,6 +49,7 @@ use crate::{
     StorageAPI,
     config::com::{CONFIG_PREFIX, read_config},
     disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
+    global::{get_global_endpoints, is_first_cluster_node_local},
     store::ECStore,
     store_api::{ObjectIO as _, ObjectOptions, PutObjReader},
 };
@@ -1118,7 +1119,15 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
                 }
                 Err(legacy_err) if is_err_config_not_found(&legacy_err) => {
                     warn!("config not found, start to init");
-                    new_and_save_tiering_config(api).await.map_err(io::Error::other)
+                    if is_first_cluster_node_local().await {
+                        new_and_save_tiering_config(api).await.map_err(io::Error::other)
+                    } else {
+                        Ok(TierConfigMgr {
+                            driver_cache: HashMap::new(),
+                            tiers: HashMap::new(),
+                            last_refreshed_at: OffsetDateTime::now_utc(),
+                        })
+                    }
                 }
                 Err(legacy_err) => Err(io::Error::other(legacy_err)),
             }
@@ -1166,7 +1175,14 @@ async fn write_tier_config_to_rustfs<S: StorageAPI>(api: Arc<S>, path: &str, dat
 pub async fn try_migrate_tiering_config<S: StorageAPI>(api: Arc<S>) {
     let target_path = tier_config_path(TIER_CONFIG_FILE);
     if api
-        .get_object_info(RUSTFS_META_BUCKET, &target_path, &ObjectOptions::default())
+        .get_object_info(
+            RUSTFS_META_BUCKET,
+            &target_path,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
         .await
         .is_ok()
     {

crates/filemeta/Cargo.toml

@@ -51,3 +51,6 @@ harness = false
 
 [lints]
 workspace = true
+
+[lib]
+doctest = false

crates/filemeta/src/filemeta.rs

@@ -74,6 +74,14 @@ pub(crate) fn data_key_for_version(version_id: Option<Uuid>) -> String {
     }
 }
 
+fn legacy_data_key_for_version(version_id: Option<Uuid>) -> Option<String> {
+    if version_id.is_none() || version_id == Some(Uuid::nil()) {
+        Some(Uuid::nil().to_string())
+    } else {
+        None
+    }
+}
+
 pub const TRANSITION_COMPLETE: &str = "complete";
 pub const TRANSITION_PENDING: &str = "pending";
 
@@ -164,6 +172,21 @@ impl FileMeta {
         });
     }
 
+    fn find_inline_data_for_version(&self, version_id: Option<Uuid>) -> Result<Option<Vec<u8>>> {
+        let key = data_key_for_version(version_id);
+        if let Some(data) = self.data.find(key.as_str())? {
+            return Ok(Some(data));
+        }
+
+        if let Some(legacy_key) = legacy_data_key_for_version(version_id)
+            && legacy_key != key
+        {
+            return self.data.find(legacy_key.as_str());
+        }
+
+        Ok(None)
+    }
+
     // Find version
     pub fn find_version(&self, vid: Option<Uuid>) -> Result<(usize, FileMetaVersion)> {
         let vid = vid.unwrap_or_default();
@@ -709,10 +732,7 @@ impl FileMeta {
             }
 
             if read_data && fi.inline_data() {
-                fi.data = self
-                    .data
-                    .find(data_key_for_version(fi.version_id).as_str())?
-                    .map(bytes::Bytes::from);
+                fi.data = self.find_inline_data_for_version(fi.version_id)?.map(bytes::Bytes::from);
             }
 
             found_fi = Some(fi);
@@ -1071,6 +1091,28 @@ mod test {
         assert!(!inline_data.is_empty(), "Inline data should not be empty");
     }
 
+    #[test]
+    fn test_into_fileinfo_reads_legacy_nil_uuid_inline_key() {
+        let mut fm = FileMeta::new();
+        let mut fi = FileInfo::new("test", 2, 1);
+        fi.mod_time = Some(OffsetDateTime::now_utc());
+        fi.data = Some(Bytes::from_static(b"legacy-inline"));
+        fi.set_inline_data();
+        fm.add_version(fi).unwrap();
+
+        let current_key = data_key_for_version(Some(Uuid::nil()));
+        let legacy_key = legacy_data_key_for_version(Some(Uuid::nil())).unwrap();
+        let payload = fm.data.find(current_key.as_str()).unwrap().unwrap();
+
+        fm.data.remove_key(current_key.as_str()).unwrap();
+        fm.data.replace(legacy_key.as_str(), payload.clone()).unwrap();
+
+        let restored = fm.into_fileinfo("bucket", "test", "", true, false, true).unwrap();
+
+        assert!(restored.inline_data());
+        assert_eq!(restored.data, Some(Bytes::from(payload)));
+    }
+
     #[test]
     fn test_error_handling_and_recovery() {
         // Test error handling and recovery

crates/filemeta/src/filemeta/inline_data.rs

@@ -43,11 +43,7 @@ impl FileMeta {
         let vid = version_id.unwrap_or_default();
 
         if self.data.entries().unwrap_or_default() > 0
-            && self
-                .data
-                .find(super::data_key_for_version(version_id).as_str())
-                .unwrap_or_default()
-                .is_some()
+            && self.find_inline_data_for_version(version_id).unwrap_or_default().is_some()
         {
             return 0;
         }

crates/heal/Cargo.toml

@@ -50,3 +50,6 @@ tracing-subscriber = { workspace = true }
 tempfile = { workspace = true }
 walkdir = { workspace = true }
 http = { workspace = true }
+
+[lib]
+doctest = false

crates/iam/Cargo.toml

@@ -58,3 +58,6 @@ url = { workspace = true }
 
 [dev-dependencies]
 pollster.workspace = true
+
+[lib]
+doctest = false

crates/iam/src/manager.rs

@@ -25,6 +25,7 @@ use crate::{
 };
 use futures::future::join_all;
 use rustfs_credentials::{Credentials, EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, get_global_action_cred};
+use rustfs_ecstore::global::is_first_cluster_node_local;
 use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq, GroupDesc};
 use rustfs_policy::{
     arn::ARN,
@@ -303,6 +304,10 @@ where
             return Ok(());
         }
 
+        if !is_first_cluster_node_local().await {
+            return Ok(());
+        }
+
         self.api.save_iam_config(IAMFormat::new_version_1(), path).await
     }
     pub async fn get_user(&self, access_key: &str) -> Option<UserIdentity> {

crates/iam/src/oidc.rs

@@ -25,6 +25,8 @@ use openidconnect::{
     PkceCodeVerifier, RedirectUrl, Scope,
 };
 use rustfs_config::oidc::*;
+use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
+use rustfs_ecstore::config::{Config as ServerConfig, KVS, get_global_server_config};
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
 use std::collections::HashMap;
@@ -101,7 +103,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
 // ---- Public types (unchanged API) ----
 
 /// Parsed configuration for a single OIDC provider.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct OidcProviderConfig {
     pub id: String,
     pub enabled: bool,
@@ -120,6 +122,26 @@ pub struct OidcProviderConfig {
     pub username_claim: String,
 }
 
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum OidcProviderConfigSource {
+    Env,
+    Persisted,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SourcedOidcProviderConfig {
+    pub config: OidcProviderConfig,
+    pub source: OidcProviderConfigSource,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct OidcProviderValidationResult {
+    pub issuer: String,
+    pub authorization_endpoint: String,
+    pub token_endpoint: Option<String>,
+}
+
 /// Summary info about a provider, returned to the console.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct OidcProviderSummary {
@@ -170,11 +192,12 @@ impl OidcSys {
     /// Parse environment variables and discover all configured OIDC providers.
     pub async fn new() -> Result<Self, String> {
         let http_client = ReqwestHttpClient(reqwest::Client::new());
-        let parsed_configs = Self::parse_env_configs();
+        let parsed_configs = load_effective_oidc_provider_configs(get_global_server_config().as_ref());
         let mut configs = HashMap::new();
         let mut provider_states = HashMap::new();
 
-        for config in parsed_configs {
+        for sourced_config in parsed_configs {
+            let config = sourced_config.config;
             if !config.enabled {
                 info!("OIDC provider '{}' is disabled, skipping", config.id);
                 continue;
@@ -620,6 +643,33 @@ impl OidcSys {
         configs
     }
 
+    fn parse_persisted_configs(cfg: &ServerConfig) -> Vec<OidcProviderConfig> {
+        let Some(subsystem) = cfg.0.get(IDENTITY_OPENID_SUB_SYS) else {
+            return Vec::new();
+        };
+
+        let mut configs = Vec::new();
+        let mut provider_ids: Vec<String> = subsystem.keys().cloned().collect();
+        provider_ids.sort();
+
+        for raw_id in provider_ids {
+            let Some(kvs) = subsystem.get(&raw_id) else {
+                continue;
+            };
+
+            let id = if raw_id == DEFAULT_DELIMITER {
+                "default"
+            } else {
+                raw_id.as_str()
+            };
+            if let Some(config) = Self::parse_single_persisted_provider(kvs, id) {
+                configs.push(config);
+            }
+        }
+
+        configs
+    }
+
     /// Parse a single provider's config from env vars with the given suffix.
     fn parse_single_provider(env_suffix: &str, id: &str) -> Option<OidcProviderConfig> {
         let get_env = |base: &str| -> String { std::env::var(format!("{base}{env_suffix}")).unwrap_or_default() };
@@ -716,6 +766,68 @@ impl OidcSys {
         })
     }
 
+    fn parse_single_persisted_provider(kvs: &KVS, id: &str) -> Option<OidcProviderConfig> {
+        let config_url = kvs.get(OIDC_CONFIG_URL);
+        if config_url.is_empty() {
+            return None;
+        }
+
+        let enabled = kvs
+            .lookup(ENABLE_KEY)
+            .unwrap_or_else(|| EnableState::Off.to_string())
+            .parse::<EnableState>()
+            .map(|s| s.is_enabled())
+            .unwrap_or(false);
+
+        let scopes_str = kvs.get(OIDC_SCOPES);
+        let scopes = if scopes_str.is_empty() {
+            OIDC_DEFAULT_SCOPES.split(',').map(String::from).collect()
+        } else {
+            scopes_str.split(',').map(|s| s.trim().to_string()).collect()
+        };
+
+        let redirect_uri_dynamic = kvs
+            .lookup(OIDC_REDIRECT_URI_DYNAMIC)
+            .unwrap_or_else(|| EnableState::On.to_string())
+            .parse::<EnableState>()
+            .map(|s| s.is_enabled())
+            .unwrap_or(true);
+
+        let claim_name = kvs
+            .lookup(OIDC_CLAIM_NAME)
+            .unwrap_or_else(|| OIDC_DEFAULT_CLAIM_NAME.to_string());
+        let groups_claim = kvs
+            .lookup(OIDC_GROUPS_CLAIM)
+            .unwrap_or_else(|| OIDC_DEFAULT_GROUPS_CLAIM.to_string());
+        let email_claim = kvs
+            .lookup(OIDC_EMAIL_CLAIM)
+            .unwrap_or_else(|| OIDC_DEFAULT_EMAIL_CLAIM.to_string());
+        let username_claim = kvs
+            .lookup(OIDC_USERNAME_CLAIM)
+            .unwrap_or_else(|| OIDC_DEFAULT_USERNAME_CLAIM.to_string());
+        let display_name = kvs.lookup(OIDC_DISPLAY_NAME).unwrap_or_else(|| id.to_string());
+        let redirect_uri = kvs.lookup(OIDC_REDIRECT_URI).filter(|v| !v.is_empty());
+        let client_secret = kvs.lookup(OIDC_CLIENT_SECRET).filter(|v| !v.is_empty());
+
+        Some(OidcProviderConfig {
+            id: id.to_string(),
+            enabled,
+            config_url,
+            client_id: kvs.get(OIDC_CLIENT_ID),
+            client_secret,
+            scopes,
+            redirect_uri,
+            redirect_uri_dynamic,
+            claim_name,
+            claim_prefix: kvs.get(OIDC_CLAIM_PREFIX),
+            role_policy: kvs.get(OIDC_ROLE_POLICY),
+            display_name,
+            groups_claim,
+            email_claim,
+            username_claim,
+        })
+    }
+
     /// Perform OIDC discovery for a provider.
     /// `discover_async` fetches the discovery document and JWKS in one step.
     async fn discover_provider(config: &OidcProviderConfig, http_client: &ReqwestHttpClient) -> Result<ProviderState, String> {
@@ -736,6 +848,64 @@ impl OidcSys {
     }
 }
 
+pub fn load_oidc_provider_configs_from_env() -> Vec<OidcProviderConfig> {
+    OidcSys::parse_env_configs()
+}
+
+pub fn load_oidc_provider_configs_from_server_config(cfg: &ServerConfig) -> Vec<OidcProviderConfig> {
+    OidcSys::parse_persisted_configs(cfg)
+}
+
+pub fn merge_oidc_provider_configs(
+    env_configs: Vec<OidcProviderConfig>,
+    persisted_configs: Vec<OidcProviderConfig>,
+) -> Vec<SourcedOidcProviderConfig> {
+    let mut effective = HashMap::new();
+
+    for config in persisted_configs {
+        effective.insert(
+            config.id.clone(),
+            SourcedOidcProviderConfig {
+                config,
+                source: OidcProviderConfigSource::Persisted,
+            },
+        );
+    }
+
+    for config in env_configs {
+        effective.insert(
+            config.id.clone(),
+            SourcedOidcProviderConfig {
+                config,
+                source: OidcProviderConfigSource::Env,
+            },
+        );
+    }
+
+    let mut configs: Vec<SourcedOidcProviderConfig> = effective.into_values().collect();
+    configs.sort_by(|lhs, rhs| lhs.config.id.cmp(&rhs.config.id));
+    configs
+}
+
+pub fn load_effective_oidc_provider_configs(server_config: Option<&ServerConfig>) -> Vec<SourcedOidcProviderConfig> {
+    let env_configs = load_oidc_provider_configs_from_env();
+    let persisted_configs = server_config
+        .map(load_oidc_provider_configs_from_server_config)
+        .unwrap_or_default();
+    merge_oidc_provider_configs(env_configs, persisted_configs)
+}
+
+pub async fn validate_oidc_provider_config(config: &OidcProviderConfig) -> Result<OidcProviderValidationResult, String> {
+    let http_client = ReqwestHttpClient(reqwest::Client::new());
+    let state = OidcSys::discover_provider(config, &http_client).await?;
+
+    Ok(OidcProviderValidationResult {
+        issuer: state.metadata.issuer().to_string(),
+        authorization_endpoint: state.metadata.authorization_endpoint().to_string(),
+        token_endpoint: state.metadata.token_endpoint().map(ToString::to_string),
+    })
+}
+
 // --- Helper functions ---
 
 fn normalize_issuer(raw: &str) -> Option<(String, String, u16, String)> {
@@ -761,14 +931,21 @@ fn normalize_config_url(config_url: &str) -> Result<String, String> {
         return Err(format!("invalid config_url scheme: {}", url.scheme()));
     }
     let host = url.host_str().ok_or_else(|| "config_url missing host".to_string())?;
-    let path = url.path().trim_end_matches('/');
+    let path = url.path();
 
-    if path.contains("/.well-known/") && !path.ends_with("/.well-known/openid-configuration") {
+    // Strip `/.well-known/openid-configuration` (with optional trailing slash) if present.
+    // Everything else is preserved exactly so the issuer URL matches the provider's discovery
+    // document (e.g. Authentik includes a trailing slash, Keycloak does not).
+    let normalized_path = path
+        .strip_suffix('/')
+        .unwrap_or(path)
+        .strip_suffix("/.well-known/openid-configuration")
+        .unwrap_or(if path == "/" { "" } else { path });
+
+    if normalized_path.contains("/.well-known/") {
         return Err("config_url uses an unsupported .well-known discovery URL".into());
     }
 
-    let normalized_path = path.strip_suffix("/.well-known/openid-configuration").unwrap_or(path);
-
     let mut issuer = format!("{}://{host}", url.scheme());
     if let Some(port) = url.port() {
         issuer.push(':');
@@ -886,20 +1063,57 @@ mod tests {
 
     #[test]
     fn test_normalize_config_url() {
+        // --- Well-known suffix stripping ---
+        // Bare well-known URL → stripped to just the host
         assert_eq!(
             normalize_config_url("https://idp.example.com/.well-known/openid-configuration").unwrap(),
             "https://idp.example.com"
         );
+        // Trailing slash after well-known suffix is also stripped
         assert_eq!(
             normalize_config_url("https://idp.example.com/.well-known/openid-configuration/").unwrap(),
             "https://idp.example.com"
         );
+        // Well-known under a sub-path (Keycloak realms)
+        assert_eq!(
+            normalize_config_url("https://keycloak.example.com/realms/myrealm/.well-known/openid-configuration").unwrap(),
+            "https://keycloak.example.com/realms/myrealm"
+        );
+
+        // --- Providers WITHOUT trailing slash (Keycloak, Auth0, Okta, Google) ---
+        assert_eq!(
+            normalize_config_url("https://keycloak.example.com/realms/myrealm").unwrap(),
+            "https://keycloak.example.com/realms/myrealm"
+        );
         assert_eq!(
             normalize_config_url("https://idp.example.com/custom/realm").unwrap(),
             "https://idp.example.com/custom/realm"
         );
+
+        // --- Providers WITH trailing slash (Authentik) ---
+        assert_eq!(
+            normalize_config_url("https://auth.example.com/application/o/myapp/").unwrap(),
+            "https://auth.example.com/application/o/myapp/"
+        );
+
+        // --- Root-level issuer (bare host) ---
+        assert_eq!(normalize_config_url("https://idp.example.com").unwrap(), "https://idp.example.com");
+        assert_eq!(normalize_config_url("https://idp.example.com/").unwrap(), "https://idp.example.com");
+
+        // --- Custom port ---
+        assert_eq!(
+            normalize_config_url("https://idp.example.com:8443/auth/realms/test").unwrap(),
+            "https://idp.example.com:8443/auth/realms/test"
+        );
+        assert_eq!(
+            normalize_config_url("http://localhost:8080/application/o/app/").unwrap(),
+            "http://localhost:8080/application/o/app/"
+        );
+
+        // --- Error cases ---
         assert!(normalize_config_url("https://idp.example.com/.well-known/invalid").is_err());
         assert!(normalize_config_url("gopher://idp.example.com").is_err());
+        assert!(normalize_config_url("not-a-url").is_err());
     }
 
     #[test]
@@ -975,6 +1189,59 @@ mod tests {
         assert!(config.is_none());
     }
 
+    #[test]
+    fn test_parse_persisted_provider_config() {
+        let mut cfg = ServerConfig::new();
+        let mut kvs = KVS(vec![
+            rustfs_ecstore::config::KV {
+                key: ENABLE_KEY.to_string(),
+                value: EnableState::Off.to_string(),
+                hidden_if_empty: false,
+            },
+            rustfs_ecstore::config::KV {
+                key: OIDC_CONFIG_URL.to_string(),
+                value: String::new(),
+                hidden_if_empty: false,
+            },
+            rustfs_ecstore::config::KV {
+                key: OIDC_CLIENT_ID.to_string(),
+                value: String::new(),
+                hidden_if_empty: false,
+            },
+        ]);
+        kvs.insert(
+            OIDC_CONFIG_URL.to_string(),
+            "https://example.com/.well-known/openid-configuration".to_string(),
+        );
+        kvs.insert(OIDC_CLIENT_ID.to_string(), "console".to_string());
+        kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+
+        cfg.0
+            .entry(IDENTITY_OPENID_SUB_SYS.to_string())
+            .or_default()
+            .insert(DEFAULT_DELIMITER.to_string(), kvs);
+
+        let parsed = OidcSys::parse_persisted_configs(&cfg);
+        assert_eq!(parsed.len(), 1);
+        assert_eq!(parsed[0].id, "default");
+        assert_eq!(parsed[0].client_id, "console");
+        assert!(parsed[0].enabled);
+    }
+
+    #[test]
+    fn test_merge_oidc_provider_configs_prefers_env() {
+        let mut persisted = test_config("default");
+        persisted.display_name = "Persisted".to_string();
+
+        let mut env = test_config("default");
+        env.display_name = "Environment".to_string();
+
+        let merged = merge_oidc_provider_configs(vec![env], vec![persisted]);
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].config.display_name, "Environment");
+        assert_eq!(merged[0].source, OidcProviderConfigSource::Env);
+    }
+
     #[test]
     fn test_oidc_sys_empty() {
         let sys = OidcSys::empty();

crates/iam/src/store/object.rs

@@ -25,7 +25,7 @@ use rustfs_ecstore::store_api::{ListOperations as _, ObjectInfoOrErr, WalkOption
 use rustfs_ecstore::{
     config::{
         RUSTFS_CONFIG_PREFIX,
-        com::{delete_config, read_config, read_config_with_metadata, save_config},
+        com::{delete_config, read_config, read_config_no_lock, read_config_with_metadata, save_config},
     },
     store::ECStore,
     store_api::{ObjectInfo, ObjectOptions},
@@ -391,7 +391,7 @@ impl ObjectStore {
         // If it doesn't exist, the system bucket or metadata is not ready.
         let probe_path = format!("{}/format.json", *IAM_CONFIG_PREFIX);
 
-        match read_config(self.object_api.clone(), &probe_path).await {
+        match read_config_no_lock(self.object_api.clone(), &probe_path).await {
             Ok(_) => Ok(()),
             Err(rustfs_ecstore::error::StorageError::ConfigNotFound) => Err(Error::other(format!(
                 "Storage metadata not ready: probe object '{}' not found (expected IAM config to be initialized)",

crates/iam/src/sys.rs

@@ -794,6 +794,15 @@ impl<T: Store> IamSys<T> {
                     return combined.is_allowed(args).await;
                 }
             }
+
+            if args.deny_only {
+                let combined_policy = Policy::default();
+                let (has_session_policy, is_allowed_sp) = is_allowed_by_session_policy(args);
+                if has_session_policy {
+                    return is_allowed_sp && combined_policy.is_allowed(args).await;
+                }
+                return combined_policy.is_allowed(args).await;
+            }
             return false;
         }
 
@@ -803,9 +812,14 @@ impl<T: Store> IamSys<T> {
             } else {
                 let (a, c) = self.store.merge_policies(&policies.join(",")).await;
                 if a.is_empty() {
-                    return false;
+                    if args.deny_only {
+                        Policy::default()
+                    } else {
+                        return false;
+                    }
+                } else {
+                    c
                 }
-                c
             }
         };
 
@@ -1035,17 +1049,20 @@ mod tests {
     use rustfs_credentials::Credentials;
     use rustfs_policy::auth::UserIdentity;
     use rustfs_policy::policy::Args;
-    use rustfs_policy::policy::action::{Action, S3Action};
+    use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
+    use serde_json::Value;
     use std::collections::HashMap;
     use time::OffsetDateTime;
 
-    /// Mock Store that populates cache in load_all so STS temp credentials
-    /// without args.groups still get group-attached policies via parent user (groups fallback).
+    /// Mock Store for STS tests: either group-attached policies via parent user, or no IAM policies.
     #[derive(Clone)]
-    struct StsGroupsFallbackMockStore;
+    struct StsTestMockStore {
+        /// When true, parent user has no groups and no mapped policies (empty `policy_db_get`).
+        empty_policies: bool,
+    }
 
     #[async_trait::async_trait]
-    impl Store for StsGroupsFallbackMockStore {
+    impl Store for StsTestMockStore {
         fn has_watcher(&self) -> bool {
             false
         }
@@ -1175,14 +1192,48 @@ mod tests {
         }
 
         async fn load_all(&self, cache: &Cache) -> Result<()> {
-            const PARENT_USER: &str = "sts-fallback-test-parent";
-            const GROUP_NAME: &str = "testgroup";
-
             let policy_docs = get_default_policyes();
             cache
                 .policy_docs
                 .store(Arc::new(CacheEntity::new(policy_docs).update_load_time()));
 
+            if self.empty_policies {
+                const PARENT_USER: &str = "sts-empty-parent-policy-test";
+                let creds = Credentials {
+                    access_key: PARENT_USER.to_string(),
+                    secret_key: "longenoughsecret".to_string(),
+                    session_token: String::new(),
+                    expiration: None,
+                    status: "on".to_string(),
+                    parent_user: String::new(),
+                    groups: None,
+                    claims: None,
+                    name: None,
+                    description: None,
+                };
+                let parent_identity = UserIdentity {
+                    version: 1,
+                    credentials: creds,
+                    update_at: Some(OffsetDateTime::now_utc()),
+                };
+                let mut users = HashMap::new();
+                users.insert(PARENT_USER.to_string(), parent_identity);
+                cache.users.store(Arc::new(CacheEntity::new(users).update_load_time()));
+
+                cache.groups.store(Arc::new(CacheEntity::default().update_load_time()));
+                cache
+                    .group_policies
+                    .store(Arc::new(CacheEntity::default().update_load_time()));
+                cache.user_policies.store(Arc::new(CacheEntity::default().update_load_time()));
+                cache.sts_accounts.store(Arc::new(CacheEntity::default().update_load_time()));
+                cache.sts_policies.store(Arc::new(CacheEntity::default().update_load_time()));
+                cache.build_user_group_memberships();
+                return Ok(());
+            }
+
+            const PARENT_USER: &str = "sts-fallback-test-parent";
+            const GROUP_NAME: &str = "testgroup";
+
             let creds = Credentials {
                 access_key: PARENT_USER.to_string(),
                 secret_key: "longenoughsecret".to_string(),
@@ -1231,7 +1282,7 @@ mod tests {
     /// would be denied.
     #[tokio::test]
     async fn test_sts_groups_fallback_temp_creds_receive_parent_group_policies() {
-        let store = StsGroupsFallbackMockStore;
+        let store = StsTestMockStore { empty_policies: false };
         let cache_manager = IamCache::new(store).await;
         let iam_sys = IamSys::new(cache_manager);
 
@@ -1257,13 +1308,93 @@ mod tests {
         );
     }
 
+    /// Regression: `deny_only` with empty IAM policies must still evaluate `sessionPolicy-extracted`
+    /// so session policy Deny cannot be bypassed (see PR #2250 review).
+    #[tokio::test]
+    async fn test_sts_deny_only_session_policy_deny_blocks_when_iam_policies_empty() {
+        let store = StsTestMockStore { empty_policies: true };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let parent_user = "sts-empty-parent-policy-test";
+        let session_policy_json = r#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Deny",
+      "Action": ["admin:CreateUser"],
+      "Resource": ["arn:aws:s3:::*"]
+    }
+  ]
+}"#;
+        let mut claims = HashMap::new();
+        claims.insert(SESSION_POLICY_NAME_EXTRACTED.to_string(), Value::String(session_policy_json.to_string()));
+        let groups: Option<Vec<String>> = None;
+        let args = Args {
+            account: parent_user,
+            groups: &groups,
+            action: Action::AdminAction(AdminAction::CreateUserAdminAction),
+            bucket: "",
+            conditions: &HashMap::new(),
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: true,
+        };
+
+        let allowed = iam_sys.is_allowed_sts(&args, parent_user).await;
+        assert!(
+            !allowed,
+            "session policy Deny must be evaluated even when IAM policies are empty and deny_only is set"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_sts_deny_only_session_policy_allow_when_no_deny_on_action() {
+        let store = StsTestMockStore { empty_policies: true };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let parent_user = "sts-empty-parent-policy-test";
+        let session_policy_json = r#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["s3:GetObject"],
+      "Resource": ["arn:aws:s3:::bucket/*"]
+    }
+  ]
+}"#;
+        let mut claims = HashMap::new();
+        claims.insert(SESSION_POLICY_NAME_EXTRACTED.to_string(), Value::String(session_policy_json.to_string()));
+        let groups: Option<Vec<String>> = None;
+        let args = Args {
+            account: parent_user,
+            groups: &groups,
+            action: Action::AdminAction(AdminAction::CreateUserAdminAction),
+            bucket: "",
+            conditions: &HashMap::new(),
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: true,
+        };
+
+        let allowed = iam_sys.is_allowed_sts(&args, parent_user).await;
+        assert!(
+            allowed,
+            "deny_only with no matching Deny in session policy should still allow self-service-style checks"
+        );
+    }
+
     /// Regression test for cross-node IAM notifications:
     /// `load_user` must populate user cache, and regular-user mapped policy must be written to
     /// `user_policies` (not `sts_policies`), otherwise list-users and bucket-scoped user listing
     /// may miss users on follower nodes.
     #[tokio::test]
     async fn test_load_user_notification_populates_user_and_policy_caches() {
-        let store = StsGroupsFallbackMockStore;
+        let store = StsTestMockStore { empty_policies: false };
         let cache_manager = IamCache::new(store).await;
         let iam_sys = IamSys::new(cache_manager);
 

crates/lock/Cargo.toml

@@ -43,3 +43,6 @@ parking_lot.workspace = true
 smallvec.workspace = true
 smartstring.workspace = true
 crossbeam-queue = { workspace = true }
+
+[lib]
+doctest = false

crates/lock/src/distributed_lock.rs

@@ -230,7 +230,21 @@ impl DistributedLock {
         } else {
             // Check if it's a timeout or quorum failure
             if let Some(error_msg) = &resp.error {
-                warn!("acquire_lock_quorum error: {}", error_msg);
+                if request.suppress_contention_logs {
+                    tracing::debug!(
+                        resource = %request.resource,
+                        owner = %request.owner,
+                        "acquire_lock_quorum contention: {}",
+                        error_msg
+                    );
+                } else {
+                    warn!(
+                        resource = %request.resource,
+                        owner = %request.owner,
+                        "acquire_lock_quorum error: {}",
+                        error_msg
+                    );
+                }
                 if error_msg.contains("quorum") {
                     // This is a quorum failure - return appropriate error
                     // Extract achieved count from error message or use individual_locks.len()
@@ -266,6 +280,21 @@ impl DistributedLock {
         self.acquire_guard(&req).await
     }
 
+    /// Convenience: acquire exclusive lock with expected contention logs suppressed
+    pub async fn lock_guard_quiet(
+        &self,
+        resource: ObjectKey,
+        owner: &str,
+        timeout: Duration,
+        ttl: Duration,
+    ) -> Result<Option<DistributedLockGuard>> {
+        let req = LockRequest::new(resource, LockType::Exclusive, owner)
+            .with_acquire_timeout(timeout)
+            .with_ttl(ttl)
+            .with_suppress_contention_logs(true);
+        self.acquire_guard(&req).await
+    }
+
     /// Convenience: acquire shared lock as a guard
     pub async fn rlock_guard(
         &self,
@@ -307,11 +336,24 @@ impl DistributedLock {
                             individual_locks.push((lock_info.id.clone(), self.clients[idx].clone()));
                         }
                     } else {
-                        tracing::warn!(
-                            "Failed to acquire lock on client from response: {}, error: {}",
-                            idx,
-                            resp.error.unwrap_or_else(|| "unknown error".to_string())
-                        );
+                        let error = resp.error.unwrap_or_else(|| "unknown error".to_string());
+                        if request.suppress_contention_logs {
+                            tracing::debug!(
+                                resource = %request.resource,
+                                owner = %request.owner,
+                                "Failed to acquire lock on client from response: {}, error: {}",
+                                idx,
+                                error
+                            );
+                        } else {
+                            tracing::warn!(
+                                resource = %request.resource,
+                                owner = %request.owner,
+                                "Failed to acquire lock on client from response: {}, error: {}",
+                                idx,
+                                error
+                            );
+                        }
                     }
                 }
                 Err(e) => {

crates/lock/src/namespace/mod.rs

@@ -58,6 +58,16 @@ impl NamespaceLockWrapper {
         self.lock.get_write_lock(self.resource.clone(), &self.owner, timeout).await
     }
 
+    /// Acquire write lock with expected contention logs suppressed
+    pub async fn get_write_lock_quiet(
+        &self,
+        timeout: Duration,
+    ) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
+        self.lock
+            .get_write_lock_quiet(self.resource.clone(), &self.owner, timeout)
+            .await
+    }
+
     /// Acquire read lock (shared lock) with timeout
     /// Returns the guard if acquisition succeeds, or an error if it fails
     pub async fn get_read_lock(&self, timeout: Duration) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
@@ -237,6 +247,29 @@ impl NamespaceLock {
         }
     }
 
+    /// Acquire write lock while suppressing expected contention warnings
+    pub async fn get_write_lock_quiet(
+        &self,
+        resource: ObjectKey,
+        owner: &str,
+        timeout: Duration,
+    ) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
+        let ttl = crate::fast_lock::DEFAULT_LOCK_TIMEOUT;
+        let resource_str = format!("{}", resource);
+        match self {
+            Self::Distributed(lock) => match lock.lock_guard_quiet(resource, owner, timeout, ttl).await {
+                Ok(Some(guard)) => Ok(NamespaceLockGuard::Standard(guard)),
+                Ok(None) => Err(crate::error::LockError::timeout(resource_str, timeout)),
+                Err(e) => Err(e),
+            },
+            Self::Local(lock) => match lock.lock_guard(resource, owner, timeout, ttl).await {
+                Ok(Some(guard)) => Ok(NamespaceLockGuard::Fast(guard)),
+                Ok(None) => Err(crate::error::LockError::timeout(resource_str, timeout)),
+                Err(e) => Err(e),
+            },
+        }
+    }
+
     /// Acquire read lock (shared lock) with timeout
     /// Returns the guard if acquisition succeeds, or an error if it fails
     pub async fn get_read_lock(

crates/lock/src/types.rs

@@ -225,6 +225,8 @@ pub struct LockRequest {
     pub priority: LockPriority,
     /// Deadlock detection
     pub deadlock_detection: bool,
+    /// Suppress warning logs for expected contention failures
+    pub suppress_contention_logs: bool,
 }
 
 impl LockRequest {
@@ -240,6 +242,7 @@ impl LockRequest {
             metadata: LockMetadata::default(),
             priority: LockPriority::default(),
             deadlock_detection: false,
+            suppress_contention_logs: false,
         }
     }
 
@@ -272,6 +275,12 @@ impl LockRequest {
         self.deadlock_detection = enabled;
         self
     }
+
+    /// Suppress warning logs for expected contention failures
+    pub fn with_suppress_contention_logs(mut self, enabled: bool) -> Self {
+        self.suppress_contention_logs = enabled;
+        self
+    }
 }
 
 /// Lock response structure

crates/madmin/Cargo.toml

@@ -35,3 +35,6 @@ hyper.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 time.workspace = true
+
+[lib]
+doctest = false

crates/madmin/src/user.rs

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as DeError};
 use serde_json::Value;
 use serde_json::value::RawValue;
 use std::collections::HashMap;
 use time::OffsetDateTime;
+use time::format_description::well_known::Rfc3339;
 
 use crate::BackendInfo;
 
@@ -492,6 +493,34 @@ impl<'de> Deserialize<'de> for SRSessionPolicy {
     }
 }
 
+fn deserialize_vec_or_default<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    Ok(Option::<Vec<String>>::deserialize(deserializer)?.unwrap_or_default())
+}
+
+fn deserialize_optional_service_account_expiration<'de, D>(deserializer: D) -> Result<Option<OffsetDateTime>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let expiration = Option::<String>::deserialize(deserializer)?;
+    let Some(expiration) = expiration else {
+        return Ok(None);
+    };
+    let expiration = expiration.trim();
+    if expiration.is_empty() {
+        return Ok(None);
+    }
+
+    let expiration = OffsetDateTime::parse(expiration, &Rfc3339).map_err(D::Error::custom)?;
+    if expiration.unix_timestamp() == 0 {
+        return Ok(None);
+    }
+
+    Ok(Some(expiration))
+}
+
 /// SRSvcAccCreate - create operation
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SRSvcAccCreate {
@@ -503,6 +532,7 @@ pub struct SRSvcAccCreate {
     #[serde(rename = "secretKey")]
     pub secret_key: String,
 
+    #[serde(default, deserialize_with = "deserialize_vec_or_default")]
     pub groups: Vec<String>,
 
     pub claims: HashMap<String, serde_json::Value>,
@@ -516,7 +546,11 @@ pub struct SRSvcAccCreate {
 
     pub description: String,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        deserialize_with = "deserialize_optional_service_account_expiration"
+    )]
     pub expiration: Option<OffsetDateTime>,
 
     #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")]
@@ -1242,4 +1276,23 @@ mod tests {
 
         assert!(long_req.validate().is_ok());
     }
+
+    #[test]
+    fn test_sr_svc_acc_create_deserialize_empty_expiration_as_none() {
+        let payload = r#"{
+            "parent": "useralpha",
+            "accessKey": "svcalpha",
+            "secretKey": "svcAlphaSecret123",
+            "groups": [],
+            "claims": {},
+            "sessionPolicy": null,
+            "status": "on",
+            "name": "uploaderKey",
+            "description": "alpha upload key",
+            "expiration": "   "
+        }"#;
+
+        let svc: SRSvcAccCreate = serde_json::from_str(payload).unwrap();
+        assert!(svc.expiration.is_none());
+    }
 }

crates/mcp/Cargo.toml

@@ -57,5 +57,6 @@ tracing-subscriber.workspace = true
 # File handling and MIME type detection
 mime_guess = { workspace = true }
 
-[dev-dependencies]
+[lib]
+doctest = false
 # Testing framework and utilities

crates/metrics/Cargo.toml

@@ -25,6 +25,11 @@ keywords.workspace = true
 categories.workspace = true
 authors.workspace = true
 
+[features]
+default = []
+gpu = ["dep:nvml-wrapper"]
+full = ["gpu"]
+
 [dependencies]
 rustfs-ecstore = { workspace = true }
 rustfs-utils = { workspace = true }
@@ -33,6 +38,8 @@ sysinfo = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "rt", "time", "macros"] }
 tokio-util = { workspace = true }
 tracing = { workspace = true }
+thiserror = { workspace = true }
+nvml-wrapper = { workspace = true, optional = true }
 
 [lints]
 workspace = true

crates/metrics/src/collectors/audit.rs

@@ -0,0 +1,109 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Audit metrics collector.
+//!
+//! Collects audit log metrics including failed messages, queue length,
+//! and total messages per target.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::audit`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::audit::*;
+use std::borrow::Cow;
+
+/// Audit target statistics for metrics collection.
+#[derive(Debug, Clone, Default)]
+pub struct AuditTargetStats {
+    /// Target identifier
+    pub target_id: String,
+    /// Number of messages that failed to send
+    pub failed_messages: u64,
+    /// Number of unsent messages in queue
+    pub queue_length: u64,
+    /// Total number of messages sent
+    pub total_messages: u64,
+}
+
+/// Collects audit metrics from the provided audit target statistics.
+///
+/// Uses the metric descriptors from `metrics_type::audit` module.
+/// Returns a vector of Prometheus metrics for audit statistics.
+pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec<PrometheusMetric> {
+    if stats.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(stats.len() * 3);
+    for stat in stats {
+        let target_id_label: Cow<'static, str> = Cow::Owned(stat.target_id.clone());
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, stat.failed_messages as f64)
+                .with_label("target_id", target_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64)
+                .with_label("target_id", target_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_MD, stat.total_messages as f64)
+                .with_label("target_id", target_id_label),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_audit_metrics() {
+        let stats = vec![
+            AuditTargetStats {
+                target_id: "target-1".to_string(),
+                failed_messages: 5,
+                queue_length: 10,
+                total_messages: 1000,
+            },
+            AuditTargetStats {
+                target_id: "target-2".to_string(),
+                failed_messages: 2,
+                queue_length: 5,
+                total_messages: 500,
+            },
+        ];
+
+        let metrics = collect_audit_metrics(&stats);
+
+        assert_eq!(metrics.len(), 6); // 2 targets * 3 metrics each
+
+        let failed = metrics
+            .iter()
+            .find(|m| m.value == 5.0 && m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1"));
+        assert!(failed.is_some());
+    }
+
+    #[test]
+    fn test_collect_audit_metrics_empty() {
+        let stats: Vec<AuditTargetStats> = vec![];
+        let metrics = collect_audit_metrics(&stats);
+        assert!(metrics.is_empty());
+    }
+}

crates/metrics/src/collectors/bucket.rs

@@ -16,90 +16,56 @@
 //!
 //! Collects usage metrics for each bucket in the cluster, including
 //! size, object counts, and quota information.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::node_bucket`
+//! to avoid duplication of metric names, types, and help text.
 
-use crate::MetricType;
 use crate::format::PrometheusMetric;
+use crate::metrics_type::node_bucket::*;
 use std::borrow::Cow;
 
-/// Usage statistics for a single bucket.
+/// Bucket statistics for metrics collection.
+///
+/// This struct provides a decoupled interface for collecting bucket metrics
+/// without depending on specific internal types. HTTP handlers should populate
+/// this struct from their available data sources.
 #[derive(Debug, Clone, Default)]
 pub struct BucketStats {
-    /// Bucket name
+    /// Name of the bucket
     pub name: String,
-    /// Total bytes used by the bucket
+    /// Total size of all objects in the bucket (bytes)
     pub size_bytes: u64,
-    /// Total number of objects in the bucket
+    /// Number of objects in the bucket
     pub objects_count: u64,
-    /// Quota limit in bytes (0 means no quota)
+    /// Quota limit for the bucket (bytes), 0 if no quota
     pub quota_bytes: u64,
 }
 
-// Static metric definitions
-const METRIC_SIZE: &str = "rustfs_bucket_usage_bytes";
-const METRIC_OBJECTS: &str = "rustfs_bucket_objects_total";
-const METRIC_QUOTA: &str = "rustfs_bucket_quota_bytes";
-
-const HELP_SIZE: &str = "Total bytes used by the bucket";
-const HELP_OBJECTS: &str = "Total number of objects in the bucket";
-const HELP_QUOTA: &str = "Quota limit in bytes for the bucket";
-
-/// Collects per-bucket usage metrics from the provided bucket statistics.
+/// Collects per-bucket metrics from the provided bucket statistics.
 ///
-/// # Metrics Produced
-///
-/// For each bucket, the following metrics are produced with a `bucket` label:
-///
-/// - `rustfs_bucket_usage_bytes`: Total bytes used by the bucket
-/// - `rustfs_bucket_objects_total`: Total number of objects in the bucket
-/// - `rustfs_bucket_quota_bytes`: Quota limit in bytes (0 if no quota configured)
-///
-/// # Arguments
-///
-/// * `buckets` - Slice of bucket statistics
-///
-/// # Example
-///
-/// ```
-/// use rustfs_metrics::collectors::{collect_bucket_metrics, BucketStats};
-///
-/// let buckets = vec![
-///     BucketStats {
-///         name: "my-bucket".to_string(),
-///         size_bytes: 1_000_000,
-///         objects_count: 100,
-///         quota_bytes: 10_000_000,
-///     },
-/// ];
-/// let metrics = collect_bucket_metrics(&buckets);
-/// assert_eq!(metrics.len(), 3); // size, objects, quota
-/// ```
-#[must_use]
-#[inline]
+/// Uses the metric descriptors from `metrics_type::node_bucket` module.
+/// Returns a vector of Prometheus metrics for all buckets.
 pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec<PrometheusMetric> {
     if buckets.is_empty() {
         return Vec::new();
     }
 
     let mut metrics = Vec::with_capacity(buckets.len() * 3);
-
     for bucket in buckets {
         let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
 
-        // Bucket size in bytes
         metrics.push(
-            PrometheusMetric::new(METRIC_SIZE, MetricType::Gauge, HELP_SIZE, bucket.size_bytes as f64)
+            PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, bucket.size_bytes as f64)
                 .with_label("bucket", bucket_label.clone()),
         );
 
-        // Object count
         metrics.push(
-            PrometheusMetric::new(METRIC_OBJECTS, MetricType::Gauge, HELP_OBJECTS, bucket.objects_count as f64)
+            PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, bucket.objects_count as f64)
                 .with_label("bucket", bucket_label.clone()),
         );
 
-        // Quota (always emit, 0 when no quota configured for consistent PromQL queries)
         metrics.push(
-            PrometheusMetric::new(METRIC_QUOTA, MetricType::Gauge, HELP_QUOTA, bucket.quota_bytes as f64)
+            PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, bucket.quota_bytes as f64)
                 .with_label("bucket", bucket_label),
         );
     }
@@ -118,27 +84,28 @@ mod tests {
             BucketStats {
                 name: "test-bucket".to_string(),
                 size_bytes: 1000,
-                objects_count: 50,
+                objects_count: 100,
                 quota_bytes: 0,
             },
             BucketStats {
-                name: "other-bucket".to_string(),
+                name: "another-bucket".to_string(),
                 size_bytes: 2000,
-                objects_count: 100,
+                objects_count: 200,
                 quota_bytes: 0,
             },
         ];
 
         let metrics = collect_bucket_metrics(&buckets);
-        report_metrics(&metrics); // This will compile and run, but we can't easily assert on the global recorder state here.
+        report_metrics(&metrics);
 
         // 2 buckets * 3 metrics each (size, objects, quota) = 6 metrics
         assert_eq!(metrics.len(), 6);
 
-        // Verify test-bucket metrics
+        // Verify test-bucket metrics have correct labels
+        let test_bucket_size_name = BUCKET_USAGE_BYTES_MD.get_full_metric_name();
         let test_bucket_size = metrics
             .iter()
-            .find(|m| m.name == METRIC_SIZE && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket"));
+            .find(|m| m.name == test_bucket_size_name && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket"));
         assert!(test_bucket_size.is_some());
         assert_eq!(test_bucket_size.map(|m| m.value), Some(1000.0));
     }
@@ -159,9 +126,13 @@ mod tests {
         assert_eq!(metrics.len(), 3);
 
         // Verify quota metric exists
-        let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA);
+        let quota_metric_name = BUCKET_QUOTA_BYTES_MD.get_full_metric_name();
+        let quota_metric = metrics.iter().find(|m| {
+            m.name == quota_metric_name
+                && m.value == 10000.0
+                && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "quota-bucket")
+        });
         assert!(quota_metric.is_some());
-        assert_eq!(quota_metric.map(|m| m.value), Some(10000.0));
     }
 
     #[test]
@@ -185,9 +156,13 @@ mod tests {
 
         // Zero quota should still produce a quota metric with value 0 for consistent PromQL queries
         assert_eq!(metrics.len(), 3);
-        let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA);
+        let quota_metric_name = BUCKET_QUOTA_BYTES_MD.get_full_metric_name();
+        let quota_metric = metrics.iter().find(|m| {
+            m.name == quota_metric_name
+                && m.value == 0.0
+                && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "no-quota-bucket")
+        });
         assert!(quota_metric.is_some());
-        assert_eq!(quota_metric.map(|m| m.value), Some(0.0));
     }
 
     #[test]

crates/metrics/src/collectors/bucket_replication.rs

@@ -13,32 +13,33 @@
 // limitations under the License.
 
 //! Bucket replication bandwidth metrics collector.
+//!
+//! Collects bandwidth metrics for bucket replication targets.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::bucket_replication`
+//! to avoid duplication of metric names, types, and help text.
 
-use crate::MetricType;
 use crate::format::PrometheusMetric;
+use crate::metrics_type::bucket_replication::{BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD};
 use std::borrow::Cow;
 
-/// Bucket replication bandwidth stats for one replication target.
+/// Bucket replication bandwidth statistics for metrics collection.
 #[derive(Debug, Clone, Default)]
 pub struct BucketReplicationBandwidthStats {
+    /// Name of the bucket
     pub bucket: String,
+    /// Target ARN for replication
     pub target_arn: String,
-    pub limit_bytes_per_sec: i64,
+    /// Configured bandwidth limit in bytes per second
+    pub limit_bytes_per_sec: u64,
+    /// Current bandwidth in bytes per second (EWMA)
     pub current_bandwidth_bytes_per_sec: f64,
 }
 
-const BUCKET_LABEL: &str = "bucket";
-const TARGET_ARN_LABEL: &str = "targetArn";
-
-const METRIC_BANDWIDTH_LIMIT: &str = "rustfs_bucket_replication_bandwidth_limit_bytes_per_second";
-const METRIC_BANDWIDTH_CURRENT: &str = "rustfs_bucket_replication_bandwidth_current_bytes_per_second";
-
-const HELP_BANDWIDTH_LIMIT: &str = "Configured bandwidth limit for replication in bytes per second";
-const HELP_BANDWIDTH_CURRENT: &str = "Current replication bandwidth in bytes per second (EWMA)";
-
-/// Collect bucket replication bandwidth metrics for Prometheus/OpenTelemetry export.
-#[must_use]
-#[inline]
+/// Collects bucket replication bandwidth metrics from the provided statistics.
+///
+/// Uses the metric descriptors from `metrics_type::bucket_replication` module.
+/// Returns a vector of Prometheus metrics for replication bandwidth.
 pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
     if stats.is_empty() {
         return Vec::new();
@@ -50,25 +51,15 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa
         let target_arn_label: Cow<'static, str> = Cow::Owned(stat.target_arn.clone());
 
         metrics.push(
-            PrometheusMetric::new(
-                METRIC_BANDWIDTH_LIMIT,
-                MetricType::Gauge,
-                HELP_BANDWIDTH_LIMIT,
-                stat.limit_bytes_per_sec as f64,
-            )
-            .with_label(BUCKET_LABEL, bucket_label.clone())
-            .with_label(TARGET_ARN_LABEL, target_arn_label.clone()),
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_LIMIT_MD, stat.limit_bytes_per_sec as f64)
+                .with_label("bucket", bucket_label.clone())
+                .with_label("targetArn", target_arn_label.clone()),
         );
 
         metrics.push(
-            PrometheusMetric::new(
-                METRIC_BANDWIDTH_CURRENT,
-                MetricType::Gauge,
-                HELP_BANDWIDTH_CURRENT,
-                stat.current_bandwidth_bytes_per_sec,
-            )
-            .with_label(BUCKET_LABEL, bucket_label)
-            .with_label(TARGET_ARN_LABEL, target_arn_label),
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_CURRENT_MD, stat.current_bandwidth_bytes_per_sec)
+                .with_label("bucket", bucket_label)
+                .with_label("targetArn", target_arn_label),
         );
     }
 
@@ -91,23 +82,27 @@ mod tests {
         let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
         assert_eq!(metrics.len(), 2);
 
-        let limit_metric = metrics.iter().find(|m| m.name == METRIC_BANDWIDTH_LIMIT);
+        let limit_metric_name = BUCKET_REPL_BANDWIDTH_LIMIT_MD.get_full_metric_name();
+        let limit_metric = metrics.iter().find(|m| {
+            m.name == limit_metric_name && m.value == 1_048_576.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
+        });
         assert!(limit_metric.is_some());
-        assert_eq!(limit_metric.map(|m| m.value), Some(1_048_576.0));
         assert!(
             limit_metric
                 .and_then(|m| {
                     m.labels
                         .iter()
-                        .find(|(k, _)| *k == TARGET_ARN_LABEL)
+                        .find(|(k, _)| *k == "targetArn")
                         .map(|(_, v)| v.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
                 })
                 .unwrap_or(false)
         );
 
-        let current_metric = metrics.iter().find(|m| m.name == METRIC_BANDWIDTH_CURRENT);
+        let current_metric_name = BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name();
+        let current_metric = metrics.iter().find(|m| {
+            m.name == current_metric_name && m.value == 204_800.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
+        });
         assert!(current_metric.is_some());
-        assert_eq!(current_metric.map(|m| m.value), Some(204_800.0));
     }
 
     #[test]

crates/metrics/src/collectors/cluster.rs

@@ -16,9 +16,12 @@
 //!
 //! Collects aggregate metrics across the entire RustFS cluster including
 //! total capacity, usage, and object counts.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::cluster`
+//! to avoid duplication of metric names, types, and help text.
 
-use crate::MetricType;
 use crate::format::PrometheusMetric;
+use crate::metrics_type::cluster::*;
 
 /// Cluster capacity and usage statistics for metrics collection.
 ///
@@ -41,88 +44,19 @@ pub struct ClusterStats {
     pub buckets_count: u64,
 }
 
-// Static metric definitions to avoid allocations
-const METRIC_RAW_CAPACITY: &str = "rustfs_cluster_capacity_raw_total_bytes";
-const METRIC_USABLE_CAPACITY: &str = "rustfs_cluster_capacity_usable_total_bytes";
-const METRIC_USED: &str = "rustfs_cluster_capacity_used_bytes";
-const METRIC_FREE: &str = "rustfs_cluster_capacity_free_bytes";
-const METRIC_OBJECTS: &str = "rustfs_cluster_objects_total";
-const METRIC_BUCKETS: &str = "rustfs_cluster_buckets_total";
-
-const HELP_RAW_CAPACITY: &str = "Total raw storage capacity in bytes across all disks";
-const HELP_USABLE_CAPACITY: &str = "Total usable storage capacity in bytes (accounting for erasure coding)";
-const HELP_USED: &str = "Total used storage capacity in bytes";
-const HELP_FREE: &str = "Total free storage capacity in bytes";
-const HELP_OBJECTS: &str = "Total number of objects in the cluster";
-const HELP_BUCKETS: &str = "Total number of buckets in the cluster";
-
-/// Number of metrics produced by this collector.
-const METRIC_COUNT: usize = 6;
-
-/// Collects cluster-wide metrics from the provided statistics.
+/// Collects cluster-wide metrics from the provided cluster statistics.
 ///
-/// # Metrics Produced
-///
-/// - `rustfs_cluster_capacity_raw_total_bytes`: Total raw storage capacity across all disks
-/// - `rustfs_cluster_capacity_usable_total_bytes`: Usable capacity after erasure coding overhead
-/// - `rustfs_cluster_capacity_used_bytes`: Currently used storage capacity
-/// - `rustfs_cluster_capacity_free_bytes`: Available free storage capacity
-/// - `rustfs_cluster_objects_total`: Total number of objects in the cluster
-/// - `rustfs_cluster_buckets_total`: Total number of buckets in the cluster
-///
-/// # Arguments
-///
-/// * `stats` - Cluster statistics containing capacity and usage data
-///
-/// # Example
-///
-/// ```
-/// use rustfs_metrics::collectors::{collect_cluster_metrics, ClusterStats};
-///
-/// let stats = ClusterStats {
-///     raw_capacity_bytes: 10_000_000_000,
-///     usable_capacity_bytes: 8_000_000_000,
-///     used_bytes: 2_000_000_000,
-///     free_bytes: 6_000_000_000,
-///     objects_count: 1000,
-///     buckets_count: 10,
-/// };
-/// let metrics = collect_cluster_metrics(&stats);
-/// assert_eq!(metrics.len(), 6);
-/// ```
-#[must_use]
-#[inline]
+/// Uses the metric descriptors from `metrics_type::cluster` module.
+/// Returns a vector of Prometheus metrics for cluster statistics.
 pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
-    let mut metrics = Vec::with_capacity(METRIC_COUNT);
-
-    metrics.push(PrometheusMetric::new(
-        METRIC_RAW_CAPACITY,
-        MetricType::Gauge,
-        HELP_RAW_CAPACITY,
-        stats.raw_capacity_bytes as f64,
-    ));
-    metrics.push(PrometheusMetric::new(
-        METRIC_USABLE_CAPACITY,
-        MetricType::Gauge,
-        HELP_USABLE_CAPACITY,
-        stats.usable_capacity_bytes as f64,
-    ));
-    metrics.push(PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, stats.used_bytes as f64));
-    metrics.push(PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, stats.free_bytes as f64));
-    metrics.push(PrometheusMetric::new(
-        METRIC_OBJECTS,
-        MetricType::Gauge,
-        HELP_OBJECTS,
-        stats.objects_count as f64,
-    ));
-    metrics.push(PrometheusMetric::new(
-        METRIC_BUCKETS,
-        MetricType::Gauge,
-        HELP_BUCKETS,
-        stats.buckets_count as f64,
-    ));
-
-    metrics
+    vec![
+        PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD, stats.raw_capacity_bytes as f64),
+        PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64),
+        PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64),
+        PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64),
+        PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, stats.objects_count as f64),
+        PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, stats.buckets_count as f64),
+    ]
 }
 
 #[cfg(test)]
@@ -147,24 +81,24 @@ mod tests {
         assert_eq!(metrics.len(), 6);
 
         // Verify raw capacity
-        let raw_capacity = metrics.iter().find(|m| m.name == METRIC_RAW_CAPACITY);
+        let raw_capacity_name = CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD.get_full_metric_name();
+        let raw_capacity = metrics.iter().find(|m| m.name == raw_capacity_name && m.value == 3000.0);
         assert!(raw_capacity.is_some());
-        assert_eq!(raw_capacity.map(|m| m.value), Some(3000.0));
 
         // Verify used capacity
-        let used = metrics.iter().find(|m| m.name == METRIC_USED);
+        let used_name = CLUSTER_CAPACITY_USED_BYTES_MD.get_full_metric_name();
+        let used = metrics.iter().find(|m| m.name == used_name && m.value == 1200.0);
         assert!(used.is_some());
-        assert_eq!(used.map(|m| m.value), Some(1200.0));
 
         // Verify object count
-        let objects = metrics.iter().find(|m| m.name == METRIC_OBJECTS);
+        let objects_name = CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name();
+        let objects = metrics.iter().find(|m| m.name == objects_name && m.value == 100.0);
         assert!(objects.is_some());
-        assert_eq!(objects.map(|m| m.value), Some(100.0));
 
         // Verify bucket count
-        let buckets = metrics.iter().find(|m| m.name == METRIC_BUCKETS);
+        let buckets_name = CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name();
+        let buckets = metrics.iter().find(|m| m.name == buckets_name && m.value == 5.0);
         assert!(buckets.is_some());
-        assert_eq!(buckets.map(|m| m.value), Some(5.0));
     }
 
     #[test]

crates/metrics/src/collectors/cluster_config.rs

@@ -0,0 +1,81 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Cluster config metrics collector.
+//!
+//! Collects cluster configuration metrics including storage class
+//! parity settings.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::cluster_config`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::cluster_config::*;
+
+/// Cluster configuration statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ClusterConfigStats {
+    /// Parity for reduced redundancy storage (RRS) class
+    pub rrs_parity: u32,
+    /// Parity for standard storage class
+    pub standard_parity: u32,
+}
+
+/// Collects cluster config metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::cluster_config` module.
+/// Returns a vector of Prometheus metrics for cluster configuration.
+pub fn collect_cluster_config_metrics(stats: &ClusterConfigStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&CONFIG_RRS_PARITY_MD, stats.rrs_parity as f64),
+        PrometheusMetric::from_descriptor(&CONFIG_STANDARD_PARITY_MD, stats.standard_parity as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_cluster_config_metrics() {
+        let stats = ClusterConfigStats {
+            rrs_parity: 2,
+            standard_parity: 4,
+        };
+
+        let metrics = collect_cluster_config_metrics(&stats);
+
+        assert_eq!(metrics.len(), 2);
+
+        let rrs = metrics.iter().find(|m| m.value == 2.0);
+        assert!(rrs.is_some());
+
+        let standard = metrics.iter().find(|m| m.value == 4.0);
+        assert!(standard.is_some());
+    }
+
+    #[test]
+    fn test_collect_cluster_config_metrics_default() {
+        let stats = ClusterConfigStats::default();
+        let metrics = collect_cluster_config_metrics(&stats);
+
+        assert_eq!(metrics.len(), 2);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/metrics/src/collectors/cluster_erasure_set.rs

@@ -0,0 +1,179 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Cluster erasure set metrics collector.
+//!
+//! Collects erasure coding set metrics including parity, quorum,
+//! drive counts, and health status.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::cluster_erasure_set::*;
+
+/// Erasure set statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ErasureSetStats {
+    /// Pool ID
+    pub pool_id: u32,
+    /// Set ID within the pool
+    pub set_id: u32,
+    /// Total number of drives in the set
+    pub size: u32,
+    /// Number of parity drives
+    pub parity: u32,
+    /// Number of data shards
+    pub data_shards: u32,
+    /// Read quorum
+    pub read_quorum: u32,
+    /// Write quorum
+    pub write_quorum: u32,
+    /// Number of online drives
+    pub online_drives_count: u32,
+    /// Number of healing drives
+    pub healing_drives_count: u32,
+    /// Health status (1=healthy, 0=unhealthy)
+    pub health: u8,
+    /// Read tolerance (number of drive failures tolerated for reads)
+    pub read_tolerance: u32,
+    /// Write tolerance (number of drive failures tolerated for writes)
+    pub write_tolerance: u32,
+    /// Read health status (1=healthy, 0=unhealthy)
+    pub read_health: u8,
+    /// Write health status (1=healthy, 0=unhealthy)
+    pub write_health: u8,
+}
+
+/// Collects erasure set metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for erasure sets.
+pub fn collect_erasure_set_metrics(stats: &[ErasureSetStats]) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(stats.len() * 12);
+
+    for stat in stats {
+        let pool_id_label = stat.pool_id.to_string();
+        let set_id_label = stat.set_id.to_string();
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_SIZE_MD, stat.size as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_PARITY_MD, stat.parity as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_DATA_SHARDS_MD, stat.data_shards as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_READ_QUORUM_MD, stat.read_quorum as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_QUORUM_MD, stat.write_quorum as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_ONLINE_DRIVES_COUNT_MD, stat.online_drives_count as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_HEALING_DRIVES_COUNT_MD, stat.healing_drives_count as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_HEALTH_MD, stat.health as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_READ_TOLERANCE_MD, stat.read_tolerance as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_TOLERANCE_MD, stat.write_tolerance as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_READ_HEALTH_MD, stat.read_health as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_HEALTH_MD, stat.write_health as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_erasure_set_metrics() {
+        let stats = vec![ErasureSetStats {
+            pool_id: 1,
+            set_id: 0,
+            size: 16,
+            parity: 4,
+            data_shards: 12,
+            read_quorum: 13,
+            write_quorum: 13,
+            online_drives_count: 16,
+            healing_drives_count: 0,
+            health: 1,
+            read_tolerance: 4,
+            write_tolerance: 4,
+            read_health: 1,
+            write_health: 1,
+        }];
+
+        let metrics = collect_erasure_set_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 12);
+
+        let size_name = ERASURE_SET_SIZE_MD.get_full_metric_name();
+        let size = metrics.iter().find(|m| m.name == size_name);
+        assert!(size.is_some());
+        assert_eq!(size.map(|m| m.value), Some(16.0));
+
+        let health_name = ERASURE_SET_HEALTH_MD.get_full_metric_name();
+        let health = metrics.iter().find(|m| m.name == health_name);
+        assert!(health.is_some());
+        assert_eq!(health.map(|m| m.value), Some(1.0));
+    }
+
+    #[test]
+    fn test_collect_erasure_set_metrics_empty() {
+        let stats: Vec<ErasureSetStats> = vec![];
+        let metrics = collect_erasure_set_metrics(&stats);
+        assert!(metrics.is_empty());
+    }
+}

crates/metrics/src/collectors/cluster_health.rs

@@ -0,0 +1,85 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Cluster health metrics collector.
+//!
+//! Collects cluster-wide health metrics including drive counts
+//! (offline, online, total).
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::cluster_health`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::cluster_health::*;
+
+/// Cluster health statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ClusterHealthStats {
+    /// Number of offline drives in the cluster
+    pub drives_offline_count: u64,
+    /// Number of online drives in the cluster
+    pub drives_online_count: u64,
+    /// Total number of drives in the cluster
+    pub drives_count: u64,
+}
+
+/// Collects cluster health metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::cluster_health` module.
+/// Returns a vector of Prometheus metrics for cluster health.
+pub fn collect_cluster_health_metrics(stats: &ClusterHealthStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&HEALTH_DRIVES_OFFLINE_COUNT_MD, stats.drives_offline_count as f64),
+        PrometheusMetric::from_descriptor(&HEALTH_DRIVES_ONLINE_COUNT_MD, stats.drives_online_count as f64),
+        PrometheusMetric::from_descriptor(&HEALTH_DRIVES_COUNT_MD, stats.drives_count as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_cluster_health_metrics() {
+        let stats = ClusterHealthStats {
+            drives_offline_count: 2,
+            drives_online_count: 18,
+            drives_count: 20,
+        };
+
+        let metrics = collect_cluster_health_metrics(&stats);
+
+        assert_eq!(metrics.len(), 3);
+
+        let offline = metrics.iter().find(|m| m.value == 2.0);
+        assert!(offline.is_some());
+
+        let online = metrics.iter().find(|m| m.value == 18.0);
+        assert!(online.is_some());
+    }
+
+    #[test]
+    fn test_collect_cluster_health_metrics_default() {
+        let stats = ClusterHealthStats::default();
+        let metrics = collect_cluster_health_metrics(&stats);
+
+        assert_eq!(metrics.len(), 3);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/metrics/src/collectors/cluster_iam.rs

@@ -0,0 +1,117 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Cluster IAM metrics collector.
+//!
+//! Collects IAM (Identity and Access Management) metrics including
+//! plugin authentication service stats and sync statistics.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::cluster_iam::*;
+
+/// IAM statistics.
+#[derive(Debug, Clone, Default)]
+pub struct IamStats {
+    /// Time in seconds since last failed authn service request
+    pub plugin_authn_service_last_fail_seconds: u64,
+    /// Time in seconds since last successful authn service request
+    pub plugin_authn_service_last_succ_seconds: u64,
+    /// Average RTT of successful requests in the last minute (ms)
+    pub plugin_authn_service_succ_avg_rtt_ms_minute: u64,
+    /// Maximum RTT of successful requests in the last minute (ms)
+    pub plugin_authn_service_succ_max_rtt_ms_minute: u64,
+    /// Total requests count in the last full minute
+    pub plugin_authn_service_total_requests_minute: u64,
+    /// Time in milliseconds since last successful IAM data sync
+    pub since_last_sync_millis: u64,
+    /// Number of failed IAM data syncs since server start
+    pub sync_failures: u64,
+    /// Number of successful IAM data syncs since server start
+    pub sync_successes: u64,
+}
+
+/// Collects IAM metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for IAM statistics.
+pub fn collect_iam_metrics(stats: &IamStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_LAST_FAIL_SECONDS_MD,
+            stats.plugin_authn_service_last_fail_seconds as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_LAST_SUCC_SECONDS_MD,
+            stats.plugin_authn_service_last_succ_seconds as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_SUCC_AVG_RTT_MS_MINUTE_MD,
+            stats.plugin_authn_service_succ_avg_rtt_ms_minute as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_SUCC_MAX_RTT_MS_MINUTE_MD,
+            stats.plugin_authn_service_succ_max_rtt_ms_minute as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_TOTAL_REQUESTS_MINUTE_MD,
+            stats.plugin_authn_service_total_requests_minute as f64,
+        ),
+        PrometheusMetric::from_descriptor(&SINCE_LAST_SYNC_MILLIS_MD, stats.since_last_sync_millis as f64),
+        PrometheusMetric::from_descriptor(&SYNC_FAILURES_MD, stats.sync_failures as f64),
+        PrometheusMetric::from_descriptor(&SYNC_SUCCESSES_MD, stats.sync_successes as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_iam_metrics() {
+        let stats = IamStats {
+            plugin_authn_service_last_fail_seconds: 3600,
+            plugin_authn_service_last_succ_seconds: 10,
+            plugin_authn_service_succ_avg_rtt_ms_minute: 50,
+            plugin_authn_service_succ_max_rtt_ms_minute: 200,
+            plugin_authn_service_total_requests_minute: 1000,
+            since_last_sync_millis: 5000,
+            sync_failures: 5,
+            sync_successes: 1000,
+        };
+
+        let metrics = collect_iam_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 8);
+
+        let sync_successes_name = SYNC_SUCCESSES_MD.get_full_metric_name();
+        let sync_successes = metrics.iter().find(|m| m.name == sync_successes_name);
+        assert!(sync_successes.is_some());
+        assert_eq!(sync_successes.map(|m| m.value), Some(1000.0));
+    }
+
+    #[test]
+    fn test_collect_iam_metrics_default() {
+        let stats = IamStats::default();
+        let metrics = collect_iam_metrics(&stats);
+
+        assert_eq!(metrics.len(), 8);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/metrics/src/collectors/cluster_usage.rs

@@ -0,0 +1,199 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Cluster usage metrics collector.
+//!
+//! Collects cluster-wide and per-bucket usage metrics including
+//! object counts, sizes, versions, and distributions.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::cluster_usage::*;
+
+/// Cluster-wide usage statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ClusterUsageStats {
+    /// Total bytes used in the cluster
+    pub total_bytes: u64,
+    /// Total number of objects
+    pub objects_count: u64,
+    /// Total number of object versions (including delete markers)
+    pub versions_count: u64,
+    /// Total number of delete markers
+    pub delete_markers_count: u64,
+    /// Object size distribution by range
+    pub object_size_distribution: Vec<(String, u64)>,
+    /// Version count distribution by range
+    pub versions_distribution: Vec<(String, u64)>,
+}
+
+/// Per-bucket usage statistics.
+#[derive(Debug, Clone, Default)]
+pub struct BucketUsageStats {
+    /// Bucket name
+    pub bucket: String,
+    /// Total bytes used in the bucket
+    pub total_bytes: u64,
+    /// Total number of objects in the bucket
+    pub objects_count: u64,
+    /// Total number of object versions (including delete markers)
+    pub versions_count: u64,
+    /// Total number of delete markers
+    pub delete_markers_count: u64,
+    /// Bucket quota in bytes (0 if no quota)
+    pub quota_bytes: u64,
+    /// Object size distribution by range
+    pub object_size_distribution: Vec<(String, u64)>,
+    /// Version count distribution by range
+    pub version_count_distribution: Vec<(String, u64)>,
+}
+
+/// Collects cluster-wide usage metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for cluster usage.
+pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(4 + stats.object_size_distribution.len() + stats.versions_distribution.len());
+
+    metrics.push(PrometheusMetric::from_descriptor(&USAGE_TOTAL_BYTES_MD, stats.total_bytes as f64));
+    metrics.push(PrometheusMetric::from_descriptor(&USAGE_OBJECTS_COUNT_MD, stats.objects_count as f64));
+    metrics.push(PrometheusMetric::from_descriptor(&USAGE_VERSIONS_COUNT_MD, stats.versions_count as f64));
+    metrics.push(PrometheusMetric::from_descriptor(
+        &USAGE_DELETE_MARKERS_COUNT_MD,
+        stats.delete_markers_count as f64,
+    ));
+
+    // Object size distribution
+    for (range, count) in &stats.object_size_distribution {
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_OBJECTS_DISTRIBUTION_MD, *count as f64)
+                .with_label_owned(RANGE_LABEL, range.clone()),
+        );
+    }
+
+    // Version distribution
+    for (range, count) in &stats.versions_distribution {
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_VERSIONS_DISTRIBUTION_MD, *count as f64)
+                .with_label_owned(RANGE_LABEL, range.clone()),
+        );
+    }
+
+    metrics
+}
+
+/// Collects per-bucket usage metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for bucket usage.
+pub fn collect_bucket_usage_metrics(stats: &[BucketUsageStats]) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::new();
+
+    for stat in stats {
+        let bucket_label = stat.bucket.clone();
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_BUCKET_TOTAL_BYTES_MD, stat.total_bytes as f64)
+                .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECTS_TOTAL_MD, stat.objects_count as f64)
+                .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_BUCKET_VERSIONS_COUNT_MD, stat.versions_count as f64)
+                .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_BUCKET_DELETE_MARKERS_COUNT_MD, stat.delete_markers_count as f64)
+                .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, stat.quota_bytes as f64)
+                .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+        );
+
+        // Object size distribution per bucket
+        for (range, count) in &stat.object_size_distribution {
+            metrics.push(
+                PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, *count as f64)
+                    .with_label_owned(RANGE_LABEL, range.clone())
+                    .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+            );
+        }
+
+        // Version count distribution per bucket
+        for (range, count) in &stat.version_count_distribution {
+            metrics.push(
+                PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, *count as f64)
+                    .with_label_owned(RANGE_LABEL, range.clone())
+                    .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+            );
+        }
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_cluster_usage_metrics() {
+        let stats = ClusterUsageStats {
+            total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
+            objects_count: 10000,
+            versions_count: 15000,
+            delete_markers_count: 500,
+            object_size_distribution: vec![
+                ("0-1KB".to_string(), 5000),
+                ("1KB-1MB".to_string(), 3000),
+                ("1MB-100MB".to_string(), 1500),
+                ("100MB+".to_string(), 500),
+            ],
+            versions_distribution: vec![("1".to_string(), 8000), ("2-5".to_string(), 1500), ("6+".to_string(), 500)],
+        };
+
+        let metrics = collect_cluster_usage_metrics(&stats);
+        report_metrics(&metrics);
+
+        // 4 base metrics + 4 size distribution + 3 version distribution = 11
+        assert_eq!(metrics.len(), 11);
+
+        let total_bytes_name = USAGE_TOTAL_BYTES_MD.get_full_metric_name();
+        let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
+        assert!(total_bytes.is_some());
+    }
+
+    #[test]
+    fn test_collect_bucket_usage_metrics() {
+        let stats = vec![BucketUsageStats {
+            bucket: "test-bucket".to_string(),
+            total_bytes: 1024 * 1024 * 1024 * 10, // 10 GB
+            objects_count: 1000,
+            versions_count: 1200,
+            delete_markers_count: 50,
+            quota_bytes: 1024 * 1024 * 1024 * 100, // 100 GB quota
+            object_size_distribution: vec![("0-1KB".to_string(), 500)],
+            version_count_distribution: vec![("1".to_string(), 800)],
+        }];
+
+        let metrics = collect_bucket_usage_metrics(&stats);
+        report_metrics(&metrics);
+
+        // 5 base metrics + 1 size distribution + 1 version distribution = 7
+        assert_eq!(metrics.len(), 7);
+    }
+}

crates/metrics/src/collectors/global.rs

@@ -12,9 +12,36 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! Global metrics collector initialization.
+//!
+//! This module provides the entry point for initializing all metrics collectors.
+//! The actual statistics collection functions are in `stats_collector.rs`.
+//!
+//! System monitoring collectors (migrated from `rustfs-obs::system`):
+//! - Process CPU metrics
+//! - Process memory metrics
+//! - Process disk I/O metrics
+//! - Process network I/O metrics
+
+use crate::collectors::stats_collector::{
+    collect_bucket_replication_bandwidth_stats, collect_bucket_stats, collect_cluster_stats, collect_disk_stats,
+    collect_process_stats,
+};
 use crate::collectors::{
-    BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, ResourceStats, collect_bucket_metrics,
-    collect_bucket_replication_bandwidth_metrics, collect_cluster_metrics, collect_node_metrics, collect_resource_metrics,
+    // System monitoring collectors (migrated from rustfs-obs::system)
+    ProcessCpuStats,
+    ProcessDiskStats,
+    ProcessMemoryStats,
+    ProcessNetworkStats,
+    collect_bucket_metrics,
+    collect_bucket_replication_bandwidth_metrics,
+    collect_cluster_metrics,
+    collect_node_metrics,
+    collect_process_cpu_metrics,
+    collect_process_disk_metrics,
+    collect_process_memory_metrics,
+    collect_process_network_metrics,
+    collect_resource_metrics,
 };
 use crate::constants::{
     DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL,
@@ -23,245 +50,82 @@ use crate::constants::{
     ENV_NODE_METRICS_INTERVAL, ENV_RESOURCE_METRICS_INTERVAL,
 };
 use crate::format::report_metrics;
-use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
-use rustfs_ecstore::data_usage::load_data_usage_from_backend;
-use rustfs_ecstore::global::get_global_bucket_monitor;
-use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
-use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
-use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
 use rustfs_utils::get_env_opt_u64;
-use std::sync::OnceLock;
-use std::time::{Duration, Instant};
-use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
+use std::borrow::Cow;
+use std::time::Duration;
+use sysinfo::{Pid, System};
 use tokio_util::sync::CancellationToken;
-use tracing::{instrument, warn};
+use tracing::warn;
 
-/// Process start time for calculating uptime.
-static PROCESS_START: OnceLock<Instant> = OnceLock::new();
+/// Default interval for system monitoring metrics (15 seconds)
+const DEFAULT_SYSTEM_METRICS_INTERVAL: Duration = Duration::from_secs(15);
+/// Environment variable for system monitoring interval
+const ENV_SYSTEM_METRICS_INTERVAL: &str = "RUSTFS_METRICS_SYSTEM_INTERVAL_SEC";
+/// Legacy environment variable for system monitoring interval
+const LEGACY_SYSTEM_METRICS_INTERVAL: &str = "RUSTFS_OBS_METRICS_SYSTEM_INTERVAL_MS";
 
-/// Get the process start time, initializing it on first call.
-#[inline]
-fn get_process_start() -> &'static Instant {
-    PROCESS_START.get_or_init(Instant::now)
-}
-
-/// Collect cluster statistics from the storage layer.
-#[instrument]
-async fn collect_cluster_stats() -> ClusterStats {
-    let Some(store) = new_object_layer_fn() else {
-        return ClusterStats::default();
-    };
-
-    let storage_info = store.storage_info().await;
-
-    let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
-    let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
-    let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
-    let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
-
-    // Get bucket and object counts from data usage info
-    let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
-        Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
-        Err(e) => {
-            warn!("Failed to load data usage from backend: {}", e);
-            // Fall back to bucket list for buckets_count, objects_count stays 0
-            let buckets = store
-                .list_bucket(&BucketOptions {
-                    cached: true,
-                    ..Default::default()
-                })
-                .await
-                .unwrap_or_else(|e| {
-                    warn!("Failed to list buckets for cluster metrics: {}", e);
-                    Vec::new()
-                });
-            (buckets.len() as u64, 0)
-        }
-    };
-
-    ClusterStats {
-        raw_capacity_bytes: raw_capacity,
-        usable_capacity_bytes: usable_capacity,
-        used_bytes: used,
-        free_bytes: free,
-        objects_count,
-        buckets_count,
-    }
-}
-
-/// Collect bucket statistics from the storage layer.
-async fn collect_bucket_stats() -> Vec<BucketStats> {
-    let Some(store) = new_object_layer_fn() else {
-        return Vec::new();
-    };
-
-    // Load data usage info from backend to get bucket sizes and object counts
-    let data_usage = match load_data_usage_from_backend(store.clone()).await {
-        Ok(info) => Some(info),
-        Err(e) => {
-            warn!("Failed to load data usage from backend for bucket metrics: {}", e);
-            None
-        }
-    };
-
-    let buckets = match store
-        .list_bucket(&BucketOptions {
-            cached: true,
-            ..Default::default()
-        })
-        .await
-    {
-        Ok(b) => b,
-        Err(e) => {
-            warn!("Failed to list buckets for metrics: {}", e);
-            return Vec::new();
-        }
-    };
-
-    // Build bucket stats with real data from DataUsageInfo
-    let mut stats = Vec::with_capacity(buckets.len());
-    for bucket in buckets {
-        if bucket.name.starts_with('.') {
-            continue;
-        }
-
-        // Get size and objects_count from data usage info
-        let (size_bytes, objects_count) = data_usage
-            .as_ref()
-            .and_then(|du| du.buckets_usage.get(&bucket.name))
-            .map(|bui| (bui.size, bui.objects_count))
-            .unwrap_or((0, 0));
-
-        // Get quota from bucket metadata
-        let quota_bytes = match get_quota_config(&bucket.name).await {
-            Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0),
-            Err(_) => 0, // No quota configured or error
-        };
-
-        stats.push(BucketStats {
-            name: bucket.name,
-            size_bytes,
-            objects_count,
-            quota_bytes,
-        });
-    }
-
-    stats
-}
-
-/// Collect bucket replication bandwidth stats from the global monitor.
-fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBandwidthStats> {
-    let Some(monitor) = get_global_bucket_monitor() else {
-        return Vec::new();
-    };
-
-    monitor
-        .get_report(|_| true)
-        .bucket_stats
-        .into_iter()
-        .map(|(opts, details)| BucketReplicationBandwidthStats {
-            bucket: opts.name,
-            target_arn: opts.replication_arn,
-            limit_bytes_per_sec: details.limit_bytes_per_sec,
-            current_bandwidth_bytes_per_sec: details.current_bandwidth_bytes_per_sec,
-        })
-        .collect()
-}
-
-/// Collect disk statistics from the storage layer.
-async fn collect_disk_stats() -> Vec<DiskStats> {
-    let Some(store) = new_object_layer_fn() else {
-        return Vec::new();
-    };
-
-    let storage_info = store.storage_info().await;
-
-    storage_info
-        .disks
-        .iter()
-        .map(|disk| DiskStats {
-            server: disk.endpoint.clone(),
-            drive: disk.drive_path.clone(),
-            total_bytes: disk.total_space,
-            used_bytes: disk.used_space,
-            free_bytes: disk.available_space,
-        })
-        .collect()
-}
-
-/// Collect resource statistics for the current process.
-///
-/// Collects:
-/// - Uptime: Calculated from process start time
-/// - Memory: Process resident set size from sysinfo
-/// - CPU: Process CPU usage percentage from sysinfo
-#[inline]
-fn collect_process_stats() -> ResourceStats {
-    let uptime_seconds = get_process_start().elapsed().as_secs();
-
-    // Use sysinfo for process metrics
-    let mut sys = System::new();
-    let pid = Pid::from_u32(std::process::id());
-    sys.refresh_processes_specifics(
-        ProcessesToUpdate::Some(&[pid]),
-        true,
-        ProcessRefreshKind::nothing().with_cpu().with_memory(),
-    );
-
-    if let Some(process) = sys.process(pid) {
-        ResourceStats {
-            cpu_percent: process.cpu_usage() as f64,
-            memory_bytes: process.memory(),
-            uptime_seconds,
-        }
-    } else {
-        // Fallback if process not found
-        ResourceStats {
-            cpu_percent: 0.0,
-            memory_bytes: 0,
-            uptime_seconds,
-        }
-    }
-}
-
-/// Initialize the metrics collection system with periodic background tasks for cluster, bucket, node, and resource metrics.
+/// Initialize all metrics collectors.
 ///
 /// This function spawns background tasks that periodically collect metrics
-/// and report them using the `metrics` crate.
+/// from various sources and report them to the metrics system.
 ///
 /// # Arguments
+/// * `token` - A `CancellationToken` that can be used to gracefully shut down
+///   all metrics collection tasks.
 ///
-/// * `token` - A cancellation token to gracefully stop the metrics collection tasks.
+/// # Environment Variables
+/// The collection intervals can be configured via environment variables:
+/// - `RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`: Cluster metrics interval in seconds (default: 60)
+/// - `RUSTFS_METRICS_BUCKET_INTERVAL_SEC`: Bucket metrics interval in seconds (default: 300)
+/// - `RUSTFS_METRICS_NODE_INTERVAL_SEC`: Node/disk metrics interval in seconds (default: 60)
+/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL_SEC`: Bucket replication bandwidth interval in seconds (default: 30)
+/// - `RUSTFS_METRICS_RESOURCE_INTERVAL_SEC`: Resource metrics interval in seconds (default: 15)
+/// - `RUSTFS_METRICS_DEFAULT_INTERVAL_SEC`: Optional global default interval in seconds.
+///
+/// Legacy interval names without `_SEC` are still accepted for backward compatibility:
+/// - `RUSTFS_METRICS_CLUSTER_INTERVAL`
+/// - `RUSTFS_METRICS_BUCKET_INTERVAL`
+/// - `RUSTFS_METRICS_NODE_INTERVAL`
+/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL`
+/// - `RUSTFS_METRICS_RESOURCE_INTERVAL`
 pub fn init_metrics_collectors(token: CancellationToken) {
-    // Initialize process start time
-    get_process_start();
+    const LEGACY_CLUSTER_INTERVAL: &str = "RUSTFS_METRICS_CLUSTER_INTERVAL";
+    const LEGACY_BUCKET_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_INTERVAL";
+    const LEGACY_NODE_INTERVAL: &str = "RUSTFS_METRICS_NODE_INTERVAL";
+    const LEGACY_REPLICATION_BANDWIDTH_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL";
+    const LEGACY_RESOURCE_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL";
+    const LEGACY_DEFAULT_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL";
 
-    // Helper closure to determine interval for a specific metric type
-    let get_interval = |env_key: &str, type_default: Duration| -> Duration {
-        // 1. Try specific env var
-        // 2. Fallback to global default env var (if set differently from hardcoded default)
-        // 3. Fallback to type specific default
+    /// Parse metrics interval from environment variables with fallback to default.
+    ///
+    /// Priority: primary_env > legacy_env > default_env > legacy_default > default_value
+    fn parse_metrics_interval(primary_env: &str, legacy_env: &str, default_interval: Duration) -> Duration {
+        get_env_opt_u64(primary_env)
+            .or_else(|| get_env_opt_u64(legacy_env))
+            .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
+            .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
+            .filter(|&v| v > 0)
+            .map(Duration::from_secs)
+            .unwrap_or(default_interval)
+    }
 
-        // Helper to check if value is valid (non-zero)
-        let is_valid = |v: u64| v > 0;
+    // Read intervals from environment or use defaults
+    let cluster_interval =
+        parse_metrics_interval(ENV_CLUSTER_METRICS_INTERVAL, LEGACY_CLUSTER_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL);
 
-        if let Some(val) = get_env_opt_u64(env_key).filter(|&v| is_valid(v)) {
-            Duration::from_secs(val)
-        } else if let Some(val) = get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL).filter(|&v| is_valid(v)) {
-            Duration::from_secs(val)
-        } else {
-            type_default
-        }
-    };
+    let bucket_interval =
+        parse_metrics_interval(ENV_BUCKET_METRICS_INTERVAL, LEGACY_BUCKET_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL);
 
-    let cluster_interval = get_interval(ENV_CLUSTER_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL);
-    let bucket_interval = get_interval(ENV_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL);
-    let bucket_replication_bandwidth_interval = get_interval(
+    let bucket_replication_bandwidth_interval = parse_metrics_interval(
         ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
+        LEGACY_REPLICATION_BANDWIDTH_INTERVAL,
         DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
     );
-    let node_interval = get_interval(ENV_NODE_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL);
-    let resource_interval = get_interval(ENV_RESOURCE_METRICS_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL);
+
+    let node_interval = parse_metrics_interval(ENV_NODE_METRICS_INTERVAL, LEGACY_NODE_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL);
+
+    let resource_interval =
+        parse_metrics_interval(ENV_RESOURCE_METRICS_INTERVAL, LEGACY_RESOURCE_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL);
 
     // Spawn task for cluster metrics
     let token_clone = token.clone();
@@ -358,4 +222,132 @@ pub fn init_metrics_collectors(token: CancellationToken) {
             }
         }
     });
+
+    // Spawn task for system monitoring metrics (migrated from rustfs-obs::system)
+    let system_interval = get_env_opt_u64(ENV_SYSTEM_METRICS_INTERVAL)
+        .or_else(|| get_env_opt_u64(LEGACY_SYSTEM_METRICS_INTERVAL).map(|ms| ms / 1000)) // Convert ms to seconds
+        .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
+        .filter(|&v| v > 0)
+        .map(Duration::from_secs)
+        .unwrap_or(DEFAULT_SYSTEM_METRICS_INTERVAL);
+
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        // Get current process PID
+        let pid = match sysinfo::get_current_pid() {
+            Ok(p) => p,
+            Err(e) => {
+                warn!("Failed to get current PID for system monitoring: {}", e);
+                return;
+            }
+        };
+
+        let mut interval = tokio::time::interval(system_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    // Collect system monitoring metrics
+                    let metrics = collect_system_monitoring_metrics(pid);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("System monitoring metrics collection cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+}
+
+/// Collect all system monitoring metrics for a process.
+///
+/// This function collects CPU, memory, disk I/O, and network I/O metrics
+/// for the specified process PID.
+///
+/// # Arguments
+/// * `pid` - The process ID to monitor
+///
+/// # Returns
+/// A vector of Prometheus metrics for the process.
+fn collect_system_monitoring_metrics(pid: Pid) -> Vec<crate::format::PrometheusMetric> {
+    let mut metrics = Vec::new();
+    let mut system = System::new();
+
+    // Refresh process information
+    system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
+
+    if let Some(process) = system.process(pid) {
+        // Create labels with process attributes
+        let labels: Vec<(&'static str, Cow<'static, str>)> = vec![
+            ("process_pid", Cow::Owned(pid.as_u32().to_string())),
+            ("process_executable_name", Cow::Owned(process.name().to_string_lossy().to_string())),
+        ];
+
+        // Collect CPU metrics
+        let cpu_stats = ProcessCpuStats {
+            usage: process.cpu_usage() as f64,
+            utilization: process.cpu_usage() as f64, // Same as usage for single process
+        };
+        metrics.extend(collect_process_cpu_metrics(&cpu_stats, Some(&labels)));
+
+        // Collect memory metrics
+        let memory_stats = ProcessMemoryStats {
+            resident: process.memory(),
+            virtual_mem: process.virtual_memory(),
+        };
+        metrics.extend(collect_process_memory_metrics(&memory_stats, Some(&labels)));
+
+        // Collect disk I/O metrics
+        let disk_usage = process.disk_usage();
+        let disk_stats = ProcessDiskStats {
+            read_bytes: disk_usage.read_bytes,
+            written_bytes: disk_usage.written_bytes,
+        };
+        metrics.extend(collect_process_disk_metrics(&disk_stats, Some(&labels)));
+
+        // Collect network I/O metrics
+        // Note: sysinfo 0.38.x provides network info via Networks new type
+        // We use Networks::new_with_refreshed_list() to get network interfaces
+        let networks = sysinfo::Networks::new_with_refreshed_list();
+        let mut total_received = 0u64;
+        let mut total_transmitted = 0u64;
+        let mut per_interface = Vec::new();
+
+        for (interface_name, data) in networks.iter() {
+            let received = data.received();
+            let transmitted = data.transmitted();
+            total_received += received;
+            total_transmitted += transmitted;
+            per_interface.push((interface_name.to_string(), received, transmitted));
+        }
+
+        let network_stats = ProcessNetworkStats {
+            total_received,
+            total_transmitted,
+            per_interface,
+        };
+        metrics.extend(collect_process_network_metrics(&network_stats, Some(&labels)));
+
+        // Collect GPU metrics (if gpu feature is enabled)
+        #[cfg(feature = "gpu")]
+        {
+            use crate::collectors::{GpuCollector, collect_gpu_metrics};
+
+            match GpuCollector::new(pid) {
+                Ok(collector) => match collector.collect() {
+                    Ok(gpu_stats) => {
+                        metrics.extend(collect_gpu_metrics(&gpu_stats, &labels));
+                    }
+                    Err(e) => {
+                        warn!("GPU metrics collection failed: {}", e);
+                    }
+                },
+                Err(e) => {
+                    warn!("GPU collector initialization failed: {}", e);
+                }
+            }
+        }
+    }
+
+    metrics
 }

crates/metrics/src/collectors/ilm.rs

@@ -0,0 +1,96 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! ILM (Information Lifecycle Management) metrics collector.
+//!
+//! Collects ILM metrics including pending tasks, active tasks,
+//! and scanned versions.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::ilm`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::ilm::*;
+
+/// ILM statistics for metrics collection.
+#[derive(Debug, Clone, Default)]
+pub struct IlmStats {
+    /// Number of pending ILM expiry tasks
+    pub expiry_pending_tasks: u64,
+    /// Number of active ILM transition tasks
+    pub transition_active_tasks: u64,
+    /// Number of pending ILM transition tasks
+    pub transition_pending_tasks: u64,
+    /// Number of missed immediate ILM transition tasks
+    pub transition_missed_immediate_tasks: u64,
+    /// Total number of object versions scanned for ILM
+    pub versions_scanned: u64,
+}
+
+/// Collects ILM metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::ilm` module.
+/// Returns a vector of Prometheus metrics for ILM statistics.
+pub fn collect_ilm_metrics(stats: &IlmStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&ILM_EXPIRY_PENDING_TASKS_MD, stats.expiry_pending_tasks as f64),
+        PrometheusMetric::from_descriptor(&ILM_TRANSITION_ACTIVE_TASKS_MD, stats.transition_active_tasks as f64),
+        PrometheusMetric::from_descriptor(&ILM_TRANSITION_PENDING_TASKS_MD, stats.transition_pending_tasks as f64),
+        PrometheusMetric::from_descriptor(
+            &ILM_TRANSITION_MISSED_IMMEDIATE_TASKS_MD,
+            stats.transition_missed_immediate_tasks as f64,
+        ),
+        PrometheusMetric::from_descriptor(&ILM_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_ilm_metrics() {
+        let stats = IlmStats {
+            expiry_pending_tasks: 100,
+            transition_active_tasks: 5,
+            transition_pending_tasks: 50,
+            transition_missed_immediate_tasks: 10,
+            versions_scanned: 1000000,
+        };
+
+        let metrics = collect_ilm_metrics(&stats);
+
+        assert_eq!(metrics.len(), 5);
+
+        let pending = metrics.iter().find(|m| m.value == 100.0);
+        assert!(pending.is_some());
+
+        let scanned = metrics.iter().find(|m| m.value == 1000000.0);
+        assert!(scanned.is_some());
+    }
+
+    #[test]
+    fn test_collect_ilm_metrics_default() {
+        let stats = IlmStats::default();
+        let metrics = collect_ilm_metrics(&stats);
+
+        assert_eq!(metrics.len(), 5);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/metrics/src/collectors/logger_webhook.rs

@@ -0,0 +1,122 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Logger webhook metrics collector.
+//!
+//! Collects webhook metrics including failed messages, queue length,
+//! and total messages per webhook target.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::logger_webhook::{
+    ENDPOINT_LABEL, NAME_LABEL, WEBHOOK_FAILED_MESSAGES_MD, WEBHOOK_QUEUE_LENGTH_MD, WEBHOOK_TOTAL_MESSAGES_MD,
+};
+use std::borrow::Cow;
+
+/// Webhook target statistics.
+#[derive(Debug, Clone, Default)]
+pub struct WebhookTargetStats {
+    /// Webhook name
+    pub name: String,
+    /// Webhook endpoint URL
+    pub endpoint: String,
+    /// Number of failed messages
+    pub failed_messages: u64,
+    /// Number of messages in queue
+    pub queue_length: u64,
+    /// Total number of messages sent
+    pub total_messages: u64,
+}
+
+/// Collects webhook metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::logger_webhook` module.
+/// Returns a vector of Prometheus metrics for webhook statistics.
+pub fn collect_webhook_metrics(stats: &[WebhookTargetStats]) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(stats.len() * 3);
+
+    for stat in stats {
+        let name_label: Cow<'static, str> = Cow::Owned(stat.name.clone());
+        let endpoint_label: Cow<'static, str> = Cow::Owned(stat.endpoint.clone());
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&WEBHOOK_FAILED_MESSAGES_MD, stat.failed_messages as f64)
+                .with_label(NAME_LABEL, name_label.clone())
+                .with_label(ENDPOINT_LABEL, endpoint_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&WEBHOOK_QUEUE_LENGTH_MD, stat.queue_length as f64)
+                .with_label(NAME_LABEL, name_label.clone())
+                .with_label(ENDPOINT_LABEL, endpoint_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&WEBHOOK_TOTAL_MESSAGES_MD, stat.total_messages as f64)
+                .with_label(NAME_LABEL, name_label.clone())
+                .with_label(ENDPOINT_LABEL, endpoint_label.clone()),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_webhook_metrics() {
+        let stats = vec![WebhookTargetStats {
+            name: "alert-webhook".to_string(),
+            endpoint: "https://hooks.example.com/alert".to_string(),
+            failed_messages: 3,
+            queue_length: 15,
+            total_messages: 5000,
+        }];
+
+        let metrics = collect_webhook_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 3);
+
+        // Verify we have metrics with the expected values
+        let failed = metrics.iter().find(|m| m.value == 3.0);
+        assert!(failed.is_some());
+
+        // Verify labels
+        let total = metrics.iter().find(|m| m.value == 5000.0);
+        assert!(total.is_some());
+        let total_metric = total.unwrap();
+        assert!(
+            total_metric
+                .labels
+                .iter()
+                .any(|(k, v)| *k == NAME_LABEL && v == "alert-webhook")
+        );
+        assert!(
+            total_metric
+                .labels
+                .iter()
+                .any(|(k, v)| *k == ENDPOINT_LABEL && v == "https://hooks.example.com/alert")
+        );
+    }
+
+    #[test]
+    fn test_collect_webhook_metrics_empty() {
+        let stats: Vec<WebhookTargetStats> = vec![];
+        let metrics = collect_webhook_metrics(&stats);
+        assert!(metrics.is_empty());
+    }
+}

crates/metrics/src/collectors/mod.rs

@@ -61,16 +61,61 @@
 //! report_metrics(&metrics);
 //! ```
 
+mod audit;
 mod bucket;
 mod bucket_replication;
 mod cluster;
+mod cluster_config;
+mod cluster_erasure_set;
+mod cluster_health;
+mod cluster_iam;
+mod cluster_usage;
 pub(crate) mod global;
+mod ilm;
+mod logger_webhook;
 mod node;
+mod notification;
+mod replication;
+mod request;
 mod resource;
+mod scanner;
+mod stats_collector;
+mod system_cpu;
+mod system_drive;
+#[cfg(feature = "gpu")]
+mod system_gpu;
+mod system_memory;
+mod system_network;
+mod system_process;
 
+pub use audit::{AuditTargetStats, collect_audit_metrics};
 pub use bucket::{BucketStats, collect_bucket_metrics};
 pub use bucket_replication::{BucketReplicationBandwidthStats, collect_bucket_replication_bandwidth_metrics};
 pub use cluster::{ClusterStats, collect_cluster_metrics};
+pub use cluster_config::{ClusterConfigStats, collect_cluster_config_metrics};
+pub use cluster_erasure_set::{ErasureSetStats, collect_erasure_set_metrics};
+pub use cluster_health::{ClusterHealthStats, collect_cluster_health_metrics};
+pub use cluster_iam::{IamStats, collect_iam_metrics};
+pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
 pub use global::init_metrics_collectors;
+pub use ilm::{IlmStats, collect_ilm_metrics};
+pub use logger_webhook::{WebhookTargetStats, collect_webhook_metrics};
 pub use node::{DiskStats, collect_node_metrics};
+pub use notification::{NotificationStats, collect_notification_metrics};
+pub use replication::{ReplicationStats, collect_replication_metrics};
+pub use request::{ApiRequestStats, collect_request_metrics};
 pub use resource::{ResourceStats, collect_resource_metrics};
+pub use scanner::{ScannerStats, collect_scanner_metrics};
+pub use system_cpu::{CpuStats, ProcessCpuStats, collect_cpu_metrics, collect_process_cpu_metrics};
+pub use system_drive::{
+    DriveCountStats, DriveDetailedStats, ProcessDiskStats, collect_drive_count_metrics, collect_drive_detailed_metrics,
+    collect_process_disk_metrics,
+};
+#[cfg(feature = "gpu")]
+pub use system_gpu::{GpuCollector, GpuError, GpuStats, collect_gpu_metrics};
+pub use system_memory::{MemoryStats, ProcessMemoryStats, collect_memory_metrics, collect_process_memory_metrics};
+pub use system_network::{NetworkStats, ProcessNetworkStats, collect_network_metrics, collect_process_network_metrics};
+pub use system_process::{
+    ProcessAttributeError, ProcessAttributes, ProcessStats, ProcessStatusType, collect_process_attributes,
+    collect_process_metrics,
+};

crates/metrics/src/collectors/node.rs

@@ -16,93 +16,61 @@
 //!
 //! Collects storage metrics for each disk/drive in the cluster,
 //! including capacity, usage, and health status.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::node_disk`
+//! to avoid duplication of metric names, types, and help text.
 
-use crate::MetricType;
 use crate::format::PrometheusMetric;
+use crate::metrics_type::node_disk::*;
 use std::borrow::Cow;
 
-/// Statistics for a single disk/drive.
+/// Disk statistics for metrics collection.
+///
+/// This struct provides a decoupled interface for collecting disk metrics
+/// without depending on specific internal types. HTTP handlers should populate
+/// this struct from their available data sources.
 #[derive(Debug, Clone, Default)]
 pub struct DiskStats {
     /// Server endpoint (e.g., "node1:9000")
     pub server: String,
     /// Drive path (e.g., "/data/disk1")
     pub drive: String,
-    /// Total capacity in bytes
+    /// Total disk capacity in bytes
     pub total_bytes: u64,
-    /// Used space in bytes
+    /// Used disk space in bytes
     pub used_bytes: u64,
-    /// Free space in bytes
+    /// Free disk space in bytes
     pub free_bytes: u64,
 }
 
-// Static metric definitions
-const METRIC_TOTAL: &str = "rustfs_node_disk_total_bytes";
-const METRIC_USED: &str = "rustfs_node_disk_used_bytes";
-const METRIC_FREE: &str = "rustfs_node_disk_free_bytes";
-
-const HELP_TOTAL: &str = "Total disk capacity in bytes";
-const HELP_USED: &str = "Used disk space in bytes";
-const HELP_FREE: &str = "Free disk space in bytes";
-
-/// Collects per-node disk metrics from the provided disk statistics.
+/// Collects per-disk metrics from the provided disk statistics.
 ///
-/// # Metrics Produced
-///
-/// For each disk, the following metrics are produced with `server` and `drive` labels:
-///
-/// - `rustfs_node_disk_total_bytes`: Total capacity of the disk
-/// - `rustfs_node_disk_used_bytes`: Used space on the disk
-/// - `rustfs_node_disk_free_bytes`: Free space on the disk
-///
-/// # Arguments
-///
-/// * `disks` - Slice of disk statistics
-///
-/// # Example
-///
-/// ```
-/// use rustfs_metrics::collectors::{collect_node_metrics, DiskStats};
-///
-/// let disks = vec![
-///     DiskStats {
-///         server: "node1:9000".to_string(),
-///         drive: "/data/disk1".to_string(),
-///         total_bytes: 1_000_000_000,
-///         used_bytes: 400_000_000,
-///         free_bytes: 600_000_000,
-///     },
-/// ];
-/// let metrics = collect_node_metrics(&disks);
-/// assert_eq!(metrics.len(), 3);
-/// ```
-#[must_use]
-#[inline]
+/// Uses the metric descriptors from `metrics_type::node_disk` module.
+/// Returns a vector of Prometheus metrics for all disks.
 pub fn collect_node_metrics(disks: &[DiskStats]) -> Vec<PrometheusMetric> {
     if disks.is_empty() {
         return Vec::new();
     }
 
     let mut metrics = Vec::with_capacity(disks.len() * 3);
-
     for disk in disks {
         let server_label: Cow<'static, str> = Cow::Owned(disk.server.clone());
         let drive_label: Cow<'static, str> = Cow::Owned(disk.drive.clone());
 
         metrics.push(
-            PrometheusMetric::new(METRIC_TOTAL, MetricType::Gauge, HELP_TOTAL, disk.total_bytes as f64)
+            PrometheusMetric::from_descriptor(&NODE_DISK_TOTAL_BYTES_MD, disk.total_bytes as f64)
                 .with_label("server", server_label.clone())
                 .with_label("drive", drive_label.clone()),
         );
 
         metrics.push(
-            PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, disk.used_bytes as f64)
+            PrometheusMetric::from_descriptor(&NODE_DISK_USED_BYTES_MD, disk.used_bytes as f64)
                 .with_label("server", server_label.clone())
                 .with_label("drive", drive_label.clone()),
         );
 
         metrics.push(
-            PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, disk.free_bytes as f64)
+            PrometheusMetric::from_descriptor(&NODE_DISK_FREE_BYTES_MD, disk.free_bytes as f64)
                 .with_label("server", server_label)
                 .with_label("drive", drive_label),
         );
@@ -140,22 +108,24 @@ mod tests {
         assert_eq!(metrics.len(), 6);
 
         // Verify node1 disk1 total bytes
+        let node1_total_name = NODE_DISK_TOTAL_BYTES_MD.get_full_metric_name();
         let node1_total = metrics.iter().find(|m| {
-            m.name == METRIC_TOTAL
+            m.name == node1_total_name
+                && m.value == 1000000.0
                 && m.labels.iter().any(|(k, v)| *k == "server" && v == "node1:9000")
                 && m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk1")
         });
         assert!(node1_total.is_some());
-        assert_eq!(node1_total.map(|m| m.value), Some(1000000.0));
 
         // Verify node2 disk2 used bytes
+        let node2_used_name = NODE_DISK_USED_BYTES_MD.get_full_metric_name();
         let node2_used = metrics.iter().find(|m| {
-            m.name == METRIC_USED
+            m.name == node2_used_name
+                && m.value == 800000.0
                 && m.labels.iter().any(|(k, v)| *k == "server" && v == "node2:9000")
                 && m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk2")
         });
         assert!(node2_used.is_some());
-        assert_eq!(node2_used.map(|m| m.value), Some(800000.0));
     }
 
     #[test]

crates/metrics/src/collectors/notification.rs

@@ -0,0 +1,91 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Notification metrics collector.
+//!
+//! Collects notification system metrics including events sent,
+//! errors, and skipped events.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::cluster_notification::{
+    NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, NOTIFICATION_EVENTS_SENT_TOTAL_MD,
+    NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD,
+};
+
+/// Notification statistics.
+#[derive(Debug, Clone, Default)]
+pub struct NotificationStats {
+    /// Number of concurrent send operations in progress
+    pub current_send_in_progress: u64,
+    /// Total number of events that encountered errors
+    pub events_errors_total: u64,
+    /// Total number of events successfully sent
+    pub events_sent_total: u64,
+    /// Total number of events skipped
+    pub events_skipped_total: u64,
+}
+
+/// Collects notification metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::cluster_notification` module.
+/// Returns a vector of Prometheus metrics for notification statistics.
+pub fn collect_notification_metrics(stats: &NotificationStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, stats.current_send_in_progress as f64),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, stats.events_errors_total as f64),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SENT_TOTAL_MD, stats.events_sent_total as f64),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD, stats.events_skipped_total as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_notification_metrics() {
+        let stats = NotificationStats {
+            current_send_in_progress: 5,
+            events_errors_total: 10,
+            events_sent_total: 10000,
+            events_skipped_total: 50,
+        };
+
+        let metrics = collect_notification_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 4);
+
+        let sent = metrics.iter().find(|m| m.value == 10000.0);
+        assert!(sent.is_some());
+
+        let errors = metrics.iter().find(|m| m.value == 10.0);
+        assert!(errors.is_some());
+    }
+
+    #[test]
+    fn test_collect_notification_metrics_default() {
+        let stats = NotificationStats::default();
+        let metrics = collect_notification_metrics(&stats);
+
+        assert_eq!(metrics.len(), 4);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/metrics/src/collectors/replication.rs

@@ -0,0 +1,107 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Replication metrics collector.
+//!
+//! Collects cluster-wide replication metrics including queue stats,
+//! data transfer rates, and worker information.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::replication::*;
+
+/// Replication statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ReplicationStats {
+    /// Number of active replication workers
+    pub active_workers: u64,
+    /// Current data transfer rate in bytes/sec
+    pub current_data_transfer_rate: f64,
+    /// Bytes queued in the last full minute
+    pub last_minute_queued_bytes: u64,
+    /// Objects queued in the last full minute
+    pub last_minute_queued_count: u64,
+    /// Maximum active workers seen since server start
+    pub max_active_workers: u64,
+    /// Maximum bytes queued since server start
+    pub max_queued_bytes: u64,
+    /// Maximum objects queued since server start
+    pub max_queued_count: u64,
+    /// Maximum data transfer rate seen since server start
+    pub max_data_transfer_rate: f64,
+    /// Objects in replication backlog in the last 5 minutes
+    pub recent_backlog_count: u64,
+}
+
+/// Collects replication metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for replication statistics.
+pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_MD, stats.active_workers as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD, stats.current_data_transfer_rate),
+        PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD, stats.last_minute_queued_bytes as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD, stats.last_minute_queued_count as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_MAX_ACTIVE_WORKERS_MD, stats.max_active_workers as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_BYTES_MD, stats.max_queued_bytes as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_COUNT_MD, stats.max_queued_count as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_MAX_DATA_TRANSFER_RATE_MD, stats.max_data_transfer_rate),
+        PrometheusMetric::from_descriptor(&REPLICATION_RECENT_BACKLOG_COUNT_MD, stats.recent_backlog_count as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_replication_metrics() {
+        let stats = ReplicationStats {
+            active_workers: 10,
+            current_data_transfer_rate: 1024.0 * 1024.0 * 5.0, // 5 MB/s
+            last_minute_queued_bytes: 1024 * 1024 * 100,       // 100 MB
+            last_minute_queued_count: 500,
+            max_active_workers: 20,
+            max_queued_bytes: 1024 * 1024 * 500, // 500 MB
+            max_queued_count: 2000,
+            max_data_transfer_rate: 1024.0 * 1024.0 * 10.0, // 10 MB/s
+            recent_backlog_count: 1500,
+        };
+
+        let metrics = collect_replication_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 9);
+
+        // Verify active workers
+        let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name();
+        let active = metrics.iter().find(|m| m.name == active_name);
+        assert!(active.is_some());
+        assert_eq!(active.map(|m| m.value), Some(10.0));
+    }
+
+    #[test]
+    fn test_collect_replication_metrics_default() {
+        let stats = ReplicationStats::default();
+        let metrics = collect_replication_metrics(&stats);
+
+        assert_eq!(metrics.len(), 9);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/metrics/src/collectors/request.rs

@@ -0,0 +1,174 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! API request metrics collector.
+//!
+//! Collects API request metrics including request counts, errors,
+//! latency, and traffic statistics.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::request::*;
+
+/// API request statistics for a specific API endpoint.
+#[derive(Debug, Clone, Default)]
+pub struct ApiRequestStats {
+    /// API name (e.g., "GetObject", "PutObject")
+    pub name: String,
+    /// Request type (e.g., "s3", "admin")
+    pub req_type: String,
+    /// Number of requests currently in flight
+    pub in_flight: u64,
+    /// Total number of requests
+    pub total: u64,
+    /// Total number of errors (4xx + 5xx)
+    pub errors_total: u64,
+    /// Total number of 5xx errors
+    pub errors_5xx: u64,
+    /// Total number of 4xx errors
+    pub errors_4xx: u64,
+    /// Total number of canceled requests
+    pub canceled: u64,
+    /// TTFB distribution by bucket (le label)
+    pub ttfb_distribution: Vec<(String, f64)>,
+    /// Bytes sent
+    pub sent_bytes: u64,
+    /// Bytes received
+    pub recv_bytes: u64,
+}
+
+/// Collects API request metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for API request statistics.
+pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::new();
+
+    for stat in stats {
+        // In-flight requests
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // Total requests
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_MD, stat.total as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // Total errors
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // 5xx errors
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // 4xx errors
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // Canceled requests
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // TTFB distribution (histogram buckets)
+        for (le, value) in &stat.ttfb_distribution {
+            metrics.push(
+                PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value)
+                    .with_label_owned(NAME_LABEL, stat.name.clone())
+                    .with_label_owned(TYPE_LABEL, stat.req_type.clone())
+                    .with_label_owned(LE_LABEL, le.clone()),
+            );
+        }
+
+        // Traffic metrics
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_TRAFFIC_SENT_BYTES_MD, stat.sent_bytes as f64)
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_TRAFFIC_RECV_BYTES_MD, stat.recv_bytes as f64)
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_request_metrics() {
+        let stats = vec![ApiRequestStats {
+            name: "GetObject".to_string(),
+            req_type: "s3".to_string(),
+            in_flight: 10,
+            total: 10000,
+            errors_total: 50,
+            errors_5xx: 10,
+            errors_4xx: 40,
+            canceled: 5,
+            ttfb_distribution: vec![
+                ("0.1".to_string(), 5000.0),
+                ("0.5".to_string(), 8000.0),
+                ("1.0".to_string(), 9500.0),
+                ("+Inf".to_string(), 10000.0),
+            ],
+            sent_bytes: 1024 * 1024 * 500, // 500 MB
+            recv_bytes: 1024 * 1024 * 100, // 100 MB
+        }];
+
+        let metrics = collect_request_metrics(&stats);
+        report_metrics(&metrics);
+
+        // 6 base metrics + 4 TTFB buckets + 2 traffic metrics = 12
+        assert_eq!(metrics.len(), 12);
+
+        let total_name = API_REQUESTS_TOTAL_MD.get_full_metric_name();
+        let total = metrics.iter().find(|m| m.name == total_name);
+        assert!(total.is_some());
+        assert_eq!(total.map(|m| m.value), Some(10000.0));
+
+        let in_flight_name = API_REQUESTS_IN_FLIGHT_TOTAL_MD.get_full_metric_name();
+        let in_flight = metrics.iter().find(|m| m.name == in_flight_name);
+        assert!(in_flight.is_some());
+        assert_eq!(in_flight.map(|m| m.value), Some(10.0));
+    }
+
+    #[test]
+    fn test_collect_request_metrics_empty() {
+        let stats: Vec<ApiRequestStats> = vec![];
+        let metrics = collect_request_metrics(&stats);
+        assert!(metrics.is_empty());
+    }
+}

crates/metrics/src/collectors/resource.rs

@@ -16,17 +16,21 @@
 //!
 //! Collects system-level metrics for the RustFS process including
 //! CPU usage, memory consumption, and process uptime.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::process_resource`
+//! to avoid duplication of metric names, types, and help text.
 
-use crate::MetricType;
 use crate::format::PrometheusMetric;
+use crate::metrics_type::process_resource::*;
 
-/// Resource statistics for a RustFS process.
+/// Resource statistics for metrics collection.
 ///
-/// This struct encapsulates the resource usage data that can be
-/// collected from the operating system for the current process.
+/// This struct provides a decoupled interface for collecting resource metrics
+/// without depending on specific internal types. HTTP handlers should populate
+/// this struct from their available data sources.
 #[derive(Debug, Clone, Default)]
 pub struct ResourceStats {
-    /// CPU usage as a percentage (0.0 to 100.0+)
+    /// CPU usage as a percentage (can exceed 100% on multi-core systems)
     pub cpu_percent: f64,
     /// Resident memory usage in bytes
     pub memory_bytes: u64,
@@ -34,65 +38,16 @@ pub struct ResourceStats {
     pub uptime_seconds: u64,
 }
 
-// Static metric definitions
-const METRIC_CPU: &str = "rustfs_process_cpu_percent";
-const METRIC_MEMORY: &str = "rustfs_process_memory_bytes";
-const METRIC_UPTIME: &str = "rustfs_process_uptime_seconds";
-
-const HELP_CPU: &str = "CPU usage of the RustFS process as a percentage";
-const HELP_MEMORY: &str = "Resident memory usage of the RustFS process in bytes";
-const HELP_UPTIME: &str = "Uptime of the RustFS process in seconds";
-
-/// Number of metrics produced by this collector.
-const METRIC_COUNT: usize = 3;
-
-/// Collects system resource metrics from the provided statistics.
+/// Collects resource metrics from the provided resource statistics.
 ///
-/// # Metrics Produced
-///
-/// - `rustfs_process_cpu_percent`: CPU usage as a percentage
-/// - `rustfs_process_memory_bytes`: Resident memory usage in bytes
-/// - `rustfs_process_uptime_seconds`: Process uptime in seconds
-///
-/// # Arguments
-///
-/// * `stats` - Resource statistics for the current process
-///
-/// # Example
-///
-/// ```
-/// use rustfs_metrics::collectors::{collect_resource_metrics, ResourceStats};
-///
-/// let stats = ResourceStats {
-///     cpu_percent: 25.5,
-///     memory_bytes: 1024 * 1024 * 512, // 512 MB
-///     uptime_seconds: 3600,
-/// };
-/// let metrics = collect_resource_metrics(&stats);
-/// assert_eq!(metrics.len(), 3);
-/// ```
-#[must_use]
-#[inline]
+/// Uses the metric descriptors from `metrics_type::process_resource` module.
+/// Returns a vector of Prometheus metrics for resource statistics.
 pub fn collect_resource_metrics(stats: &ResourceStats) -> Vec<PrometheusMetric> {
-    let mut metrics = Vec::with_capacity(METRIC_COUNT);
-
-    metrics.push(PrometheusMetric::new(METRIC_CPU, MetricType::Gauge, HELP_CPU, stats.cpu_percent));
-
-    metrics.push(PrometheusMetric::new(
-        METRIC_MEMORY,
-        MetricType::Gauge,
-        HELP_MEMORY,
-        stats.memory_bytes as f64,
-    ));
-
-    metrics.push(PrometheusMetric::new(
-        METRIC_UPTIME,
-        MetricType::Gauge,
-        HELP_UPTIME,
-        stats.uptime_seconds as f64,
-    ));
-
-    metrics
+    vec![
+        PrometheusMetric::from_descriptor(&PROCESS_CPU_PERCENT_MD, stats.cpu_percent),
+        PrometheusMetric::from_descriptor(&PROCESS_MEMORY_BYTES_MD, stats.memory_bytes as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64),
+    ]
 }
 
 #[cfg(test)]
@@ -114,19 +69,21 @@ mod tests {
         assert_eq!(metrics.len(), 3);
 
         // Verify CPU metric
-        let cpu = metrics.iter().find(|m| m.name == METRIC_CPU);
+        let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name();
+        let cpu = metrics.iter().find(|m| m.name == cpu_metric_name && m.value == 45.5);
         assert!(cpu.is_some());
-        assert_eq!(cpu.map(|m| m.value), Some(45.5));
 
         // Verify memory metric
-        let memory = metrics.iter().find(|m| m.name == METRIC_MEMORY);
+        let memory_metric_name = PROCESS_MEMORY_BYTES_MD.get_full_metric_name();
+        let memory = metrics
+            .iter()
+            .find(|m| m.name == memory_metric_name && m.value == (1024 * 1024 * 256) as f64);
         assert!(memory.is_some());
-        assert_eq!(memory.map(|m| m.value), Some((1024 * 1024 * 256) as f64));
 
         // Verify uptime metric
-        let uptime = metrics.iter().find(|m| m.name == METRIC_UPTIME);
+        let uptime_metric_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name();
+        let uptime = metrics.iter().find(|m| m.name == uptime_metric_name && m.value == 7200.0);
         assert!(uptime.is_some());
-        assert_eq!(uptime.map(|m| m.value), Some(7200.0));
     }
 
     #[test]
@@ -155,9 +112,9 @@ mod tests {
         let metrics = collect_resource_metrics(&stats);
         report_metrics(&metrics);
 
-        let cpu = metrics.iter().find(|m| m.name == METRIC_CPU);
+        let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name();
+        let cpu = metrics.iter().find(|m| m.name == cpu_metric_name && m.value == 150.0);
         assert!(cpu.is_some());
-        assert_eq!(cpu.map(|m| m.value), Some(150.0));
     }
 
     #[test]

crates/metrics/src/collectors/scanner.rs

@@ -0,0 +1,99 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Scanner metrics collector.
+//!
+//! Collects background scanner metrics including bucket scans,
+//! directory scans, and object scans.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::scanner::{
+    SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_DIRECTORIES_SCANNED_MD,
+    SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
+};
+
+/// Scanner statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ScannerStats {
+    /// Number of bucket scans finished
+    pub bucket_scans_finished: u64,
+    /// Number of bucket scans started
+    pub bucket_scans_started: u64,
+    /// Number of directories scanned
+    pub directories_scanned: u64,
+    /// Number of objects scanned
+    pub objects_scanned: u64,
+    /// Number of object versions scanned
+    pub versions_scanned: u64,
+    /// Seconds since last scanner activity
+    pub last_activity_seconds: u64,
+}
+
+/// Collects scanner metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::scanner` module.
+/// Returns a vector of Prometheus metrics for scanner statistics.
+pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_scanner_metrics() {
+        let stats = ScannerStats {
+            bucket_scans_finished: 100,
+            bucket_scans_started: 100,
+            directories_scanned: 50000,
+            objects_scanned: 1000000,
+            versions_scanned: 1500000,
+            last_activity_seconds: 30,
+        };
+
+        let metrics = collect_scanner_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 6);
+
+        let objects = metrics.iter().find(|m| m.value == 1000000.0);
+        assert!(objects.is_some());
+
+        let last_activity = metrics.iter().find(|m| m.value == 30.0);
+        assert!(last_activity.is_some());
+    }
+
+    #[test]
+    fn test_collect_scanner_metrics_default() {
+        let stats = ScannerStats::default();
+        let metrics = collect_scanner_metrics(&stats);
+
+        assert_eq!(metrics.len(), 6);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/metrics/src/collectors/stats_collector.rs

@@ -0,0 +1,233 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Statistics collection functions for metrics.
+//!
+//! This module contains functions that collect statistics from various
+//! RustFS internal sources (storage layer, bucket monitor, system info)
+//! and convert them to the Stats structs used by collectors.
+
+use crate::collectors::{BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, ResourceStats};
+use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
+use rustfs_ecstore::data_usage::load_data_usage_from_backend;
+use rustfs_ecstore::global::get_global_bucket_monitor;
+use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
+use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
+use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
+use std::sync::OnceLock;
+use std::time::Instant;
+use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
+use tracing::{instrument, warn};
+
+/// Process start time for calculating uptime.
+static PROCESS_START: OnceLock<Instant> = OnceLock::new();
+
+/// Get the process start time, initializing it on first call.
+#[inline]
+fn get_process_start() -> &'static Instant {
+    PROCESS_START.get_or_init(Instant::now)
+}
+
+/// Collect cluster statistics from the storage layer.
+#[instrument]
+pub async fn collect_cluster_stats() -> ClusterStats {
+    let Some(store) = new_object_layer_fn() else {
+        return ClusterStats::default();
+    };
+
+    let storage_info = store.storage_info().await;
+
+    let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
+    let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
+    let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
+    let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
+
+    // Get bucket and object counts from data usage info
+    let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
+        Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
+        Err(e) => {
+            warn!("Failed to load data usage from backend: {}", e);
+            // Fall back to bucket list for buckets_count, objects_count stays 0
+            let buckets = store
+                .list_bucket(&BucketOptions {
+                    cached: true,
+                    ..Default::default()
+                })
+                .await
+                .unwrap_or_else(|e| {
+                    warn!("Failed to list buckets for cluster metrics: {}", e);
+                    Vec::new()
+                });
+            (buckets.len() as u64, 0)
+        }
+    };
+
+    ClusterStats {
+        raw_capacity_bytes: raw_capacity,
+        usable_capacity_bytes: usable_capacity,
+        used_bytes: used,
+        free_bytes: free,
+        objects_count,
+        buckets_count,
+    }
+}
+
+/// Collect bucket statistics from the storage layer.
+pub async fn collect_bucket_stats() -> Vec<BucketStats> {
+    let Some(store) = new_object_layer_fn() else {
+        return Vec::new();
+    };
+
+    // Load data usage info from backend to get bucket sizes and object counts
+    let data_usage = match load_data_usage_from_backend(store.clone()).await {
+        Ok(info) => Some(info),
+        Err(e) => {
+            warn!("Failed to load data usage for bucket metrics: {}", e);
+            None
+        }
+    };
+
+    // List all buckets
+    let buckets = match store
+        .list_bucket(&BucketOptions {
+            cached: true,
+            ..Default::default()
+        })
+        .await
+    {
+        Ok(buckets) => buckets,
+        Err(e) => {
+            warn!("Failed to list buckets for bucket metrics: {}", e);
+            return Vec::new();
+        }
+    };
+
+    let mut stats = Vec::with_capacity(buckets.len());
+
+    for bucket in buckets {
+        if bucket.name.starts_with('.') {
+            continue;
+        }
+
+        // Get size and objects_count from data usage info
+        let (size_bytes, objects_count) = data_usage
+            .as_ref()
+            .and_then(|du| du.buckets_usage.get(&bucket.name))
+            .map(|bui| (bui.size, bui.objects_count))
+            .unwrap_or((0, 0));
+
+        // Get quota from bucket metadata
+        let quota_bytes = match get_quota_config(&bucket.name).await {
+            Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0),
+            Err(_) => 0, // No quota configured or error
+        };
+
+        stats.push(BucketStats {
+            name: bucket.name,
+            size_bytes,
+            objects_count,
+            quota_bytes,
+        });
+    }
+
+    stats
+}
+
+/// Collect bucket replication bandwidth stats from the global monitor.
+pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBandwidthStats> {
+    let Some(monitor) = get_global_bucket_monitor() else {
+        return Vec::new();
+    };
+
+    monitor
+        .get_report(|_| true)
+        .bucket_stats
+        .into_iter()
+        .map(|(opts, details)| {
+            let target_arn = opts.replication_arn;
+            let limit_bytes_per_sec = u64::try_from(details.limit_bytes_per_sec).unwrap_or_else(|_| {
+                warn!(
+                    "Invalid bandwidth limit value for target {:?}: {}",
+                    target_arn, details.limit_bytes_per_sec
+                );
+                0
+            });
+
+            BucketReplicationBandwidthStats {
+                bucket: opts.name,
+                target_arn,
+                limit_bytes_per_sec,
+                current_bandwidth_bytes_per_sec: details.current_bandwidth_bytes_per_sec,
+            }
+        })
+        .collect()
+}
+
+/// Collect disk statistics from the storage layer.
+pub async fn collect_disk_stats() -> Vec<DiskStats> {
+    let Some(store) = new_object_layer_fn() else {
+        return Vec::new();
+    };
+
+    let storage_info = store.storage_info().await;
+
+    storage_info
+        .disks
+        .iter()
+        .map(|disk| DiskStats {
+            server: disk.endpoint.clone(),
+            drive: disk.drive_path.clone(),
+            total_bytes: disk.total_space,
+            used_bytes: disk.used_space,
+            free_bytes: disk.available_space,
+        })
+        .collect()
+}
+
+/// Collect resource statistics for the current process.
+///
+/// Collects:
+/// - Uptime: Calculated from process start time
+/// - Memory: Process resident set size from sysinfo
+/// - CPU: Process CPU usage percentage from sysinfo
+#[inline]
+pub fn collect_process_stats() -> ResourceStats {
+    let uptime_seconds = get_process_start().elapsed().as_secs();
+
+    // Use sysinfo for process metrics
+    let mut sys = System::new();
+    let pid = Pid::from_u32(std::process::id());
+    sys.refresh_processes_specifics(
+        ProcessesToUpdate::Some(&[pid]),
+        true,
+        ProcessRefreshKind::nothing().with_cpu().with_memory(),
+    );
+
+    if let Some(process) = sys.process(pid) {
+        ResourceStats {
+            cpu_percent: process.cpu_usage() as f64,
+            memory_bytes: process.memory(),
+            uptime_seconds,
+        }
+    } else {
+        // Fallback if process info unavailable
+        ResourceStats {
+            cpu_percent: 0.0,
+            memory_bytes: 0,
+            uptime_seconds,
+        }
+    }
+}

crates/metrics/src/collectors/system_cpu.rs

@@ -0,0 +1,184 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! System CPU metrics collector.
+//!
+//! Collects CPU metrics including load average, CPU time distribution,
+//! and process-level CPU usage.
+//!
+//! This module provides both system-level and process-level CPU metrics,
+//! with process-level metrics migrated from `rustfs-obs::system`.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::system_cpu::*;
+use crate::metrics_type::system_process::{PROCESS_CPU_USAGE_MD, PROCESS_CPU_UTILIZATION_MD};
+use std::borrow::Cow;
+
+/// System CPU statistics.
+#[derive(Debug, Clone, Default)]
+pub struct CpuStats {
+    /// Average CPU idle time (percentage, 0-100)
+    pub avg_idle: f64,
+    /// Average CPU I/O wait time (percentage, 0-100)
+    pub avg_iowait: f64,
+    /// CPU load average over 1 minute
+    pub load_avg: f64,
+    /// CPU load average as percentage
+    pub load_avg_perc: f64,
+    /// CPU nice time (percentage, 0-100)
+    pub nice: f64,
+    /// CPU steal time (percentage, 0-100)
+    pub steal: f64,
+    /// CPU system time (percentage, 0-100)
+    pub system: f64,
+    /// CPU user time (percentage, 0-100)
+    pub user: f64,
+}
+
+/// Process CPU statistics.
+///
+/// Contains CPU usage metrics for a specific process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessCpuStats {
+    /// CPU usage percentage (0-100)
+    pub usage: f64,
+    /// CPU utilization percentage (considering multiple cores, can exceed 100)
+    pub utilization: f64,
+}
+
+/// Collects CPU metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::system_cpu` module.
+/// Returns a vector of Prometheus metrics for CPU statistics.
+pub fn collect_cpu_metrics(stats: &CpuStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IDLE_MD, stats.avg_idle),
+        PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IOWAIT_MD, stats.avg_iowait),
+        PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_MD, stats.load_avg),
+        PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_PERC_MD, stats.load_avg_perc),
+        PrometheusMetric::from_descriptor(&SYS_CPU_NICE_MD, stats.nice),
+        PrometheusMetric::from_descriptor(&SYS_CPU_STEAL_MD, stats.steal),
+        PrometheusMetric::from_descriptor(&SYS_CPU_SYSTEM_MD, stats.system),
+        PrometheusMetric::from_descriptor(&SYS_CPU_USER_MD, stats.user),
+    ]
+}
+
+/// Collects process CPU metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::system_process` module.
+/// Returns a vector of Prometheus metrics for process CPU statistics.
+///
+/// # Arguments
+///
+/// * `stats` - Process CPU statistics
+/// * `labels` - Optional additional labels (e.g., process attributes)
+pub fn collect_process_cpu_metrics(
+    stats: &ProcessCpuStats,
+    labels: Option<&[(&'static str, Cow<'static, str>)]>,
+) -> Vec<PrometheusMetric> {
+    let mut usage_metric = PrometheusMetric::from_descriptor(&PROCESS_CPU_USAGE_MD, stats.usage);
+    let mut utilization_metric = PrometheusMetric::from_descriptor(&PROCESS_CPU_UTILIZATION_MD, stats.utilization);
+
+    if let Some(l) = labels {
+        usage_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        utilization_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+    }
+
+    vec![usage_metric, utilization_metric]
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_cpu_metrics() {
+        let stats = CpuStats {
+            avg_idle: 75.5,
+            avg_iowait: 2.3,
+            load_avg: 1.5,
+            load_avg_perc: 37.5,
+            nice: 0.5,
+            steal: 0.1,
+            system: 10.0,
+            user: 15.0,
+        };
+
+        let metrics = collect_cpu_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 8);
+
+        // Verify that metric names are properly generated from descriptors
+        assert!(metrics.iter().all(|m| m.name.starts_with("gauge.rustfs_system_cpu_")));
+    }
+
+    #[test]
+    fn test_collect_cpu_metrics_default() {
+        let stats = CpuStats::default();
+        let metrics = collect_cpu_metrics(&stats);
+
+        assert_eq!(metrics.len(), 8);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+
+    #[test]
+    fn test_collect_process_cpu_metrics() {
+        let stats = ProcessCpuStats {
+            usage: 45.5,
+            utilization: 182.0, // 4 cores at ~45% each
+        };
+
+        let metrics = collect_process_cpu_metrics(&stats, None);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 2);
+
+        // Verify usage metric
+        let usage_metric = metrics.iter().find(|m| m.name.contains("cpu_usage"));
+        assert!(usage_metric.is_some());
+        assert_eq!(usage_metric.map(|m| m.value), Some(45.5));
+
+        // Verify utilization metric
+        let util_metric = metrics.iter().find(|m| m.name.contains("cpu_utilization"));
+        assert!(util_metric.is_some());
+        assert_eq!(util_metric.map(|m| m.value), Some(182.0));
+    }
+
+    #[test]
+    fn test_collect_process_cpu_metrics_with_labels() {
+        let stats = ProcessCpuStats {
+            usage: 25.0,
+            utilization: 100.0,
+        };
+
+        let labels = vec![
+            ("process_pid", Cow::Borrowed("12345")),
+            ("process_executable_name", Cow::Borrowed("rustfs")),
+        ];
+
+        let metrics = collect_process_cpu_metrics(&stats, Some(&labels));
+        assert_eq!(metrics.len(), 2);
+
+        // All metrics should have the labels
+        for metric in &metrics {
+            assert_eq!(metric.labels.len(), 2);
+        }
+    }
+}

crates/metrics/src/collectors/system_drive.rs

@@ -0,0 +1,271 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! System drive metrics collector.
+//!
+//! Collects detailed drive/disk metrics including capacity, I/O statistics,
+//! error counts, and health status.
+//!
+//! This module provides both system-level and process-level disk metrics,
+//! with process-level metrics migrated from `rustfs-obs::system`.
+
+use crate::format::PrometheusMetric;
+use crate::metrics_type::system_drive::*;
+use crate::metrics_type::system_process::PROCESS_DISK_IO_MD;
+use std::borrow::Cow;
+
+/// Detailed drive statistics for a single drive.
+#[derive(Debug, Clone, Default)]
+pub struct DriveDetailedStats {
+    /// Server identifier (e.g., "node1:9000")
+    pub server: String,
+    /// Drive path (e.g., "/data/disk1")
+    pub drive: String,
+    /// Total capacity in bytes
+    pub total_bytes: u64,
+    /// Used capacity in bytes
+    pub used_bytes: u64,
+    /// Free capacity in bytes
+    pub free_bytes: u64,
+    /// Used inodes
+    pub used_inodes: u64,
+    /// Free inodes
+    pub free_inodes: u64,
+    /// Total inodes
+    pub total_inodes: u64,
+    /// Total timeout errors
+    pub timeout_errors_total: u64,
+    /// Total I/O errors
+    pub io_errors_total: u64,
+    /// Total availability errors
+    pub availability_errors_total: u64,
+    /// Number of I/O operations waiting
+    pub waiting_io: u64,
+    /// API latency in microseconds
+    pub api_latency_micros: u64,
+    /// Health status (1=healthy, 0=unhealthy)
+    pub health: u8,
+    /// Reads per second
+    pub reads_per_sec: f64,
+    /// Kilobytes read per second
+    pub reads_kb_per_sec: f64,
+    /// Average read await time
+    pub reads_await: f64,
+    /// Writes per second
+    pub writes_per_sec: f64,
+    /// Kilobytes written per second
+    pub writes_kb_per_sec: f64,
+    /// Average write await time
+    pub writes_await: f64,
+    /// Percentage utilization
+    pub perc_util: f64,
+}
+
+/// Aggregate drive count statistics.
+#[derive(Debug, Clone, Default)]
+pub struct DriveCountStats {
+    /// Number of offline Drives
+    pub offline_count: u64,
+    /// Number of online drives
+    pub online_count: u64,
+    /// Total number of drives
+    pub total_count: u64,
+}
+
+/// Collects detailed drive metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for each drive.
+pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<PrometheusMetric> {
+    fn push_drive_metric(
+        metrics: &mut Vec<PrometheusMetric>,
+        descriptor: &'static crate::MetricDescriptor,
+        value: f64,
+        server_label: &str,
+        drive_label: &str,
+    ) {
+        metrics.push(
+            PrometheusMetric::from_descriptor(descriptor, value)
+                .with_label_owned(DRIVE_LABEL, drive_label.to_string())
+                .with_label_owned(SERVER_LABEL, server_label.to_string()),
+        );
+    }
+
+    let mut metrics = Vec::with_capacity(stats.len() * 19);
+
+    for stat in stats {
+        let server_label = stat.server.as_str();
+        let drive_label = stat.drive.as_str();
+
+        push_drive_metric(&mut metrics, &DRIVE_TOTAL_BYTES_MD, stat.total_bytes as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_USED_BYTES_MD, stat.used_bytes as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_FREE_BYTES_MD, stat.free_bytes as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_USED_INODES_MD, stat.used_inodes as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_FREE_INODES_MD, stat.free_inodes as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_TOTAL_INODES_MD, stat.total_inodes as f64, server_label, drive_label);
+        push_drive_metric(
+            &mut metrics,
+            &DRIVE_TIMEOUT_ERRORS_MD,
+            stat.timeout_errors_total as f64,
+            server_label,
+            drive_label,
+        );
+        push_drive_metric(&mut metrics, &DRIVE_IO_ERRORS_MD, stat.io_errors_total as f64, server_label, drive_label);
+        push_drive_metric(
+            &mut metrics,
+            &DRIVE_AVAILABILITY_ERRORS_MD,
+            stat.availability_errors_total as f64,
+            server_label,
+            drive_label,
+        );
+        push_drive_metric(&mut metrics, &DRIVE_WAITING_IO_MD, stat.waiting_io as f64, server_label, drive_label);
+        push_drive_metric(
+            &mut metrics,
+            &DRIVE_API_LATENCY_MD,
+            stat.api_latency_micros as f64,
+            server_label,
+            drive_label,
+        );
+        push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.health as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_READS_PER_SEC_MD, stat.reads_per_sec, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_READS_KB_PER_SEC_MD, stat.reads_kb_per_sec, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_READS_AWAIT_MD, stat.reads_await, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_WRITES_PER_SEC_MD, stat.writes_per_sec, server_label, drive_label);
+        push_drive_metric(
+            &mut metrics,
+            &DRIVE_WRITES_KB_PER_SEC_MD,
+            stat.writes_kb_per_sec,
+            server_label,
+            drive_label,
+        );
+        push_drive_metric(&mut metrics, &DRIVE_WRITES_AWAIT_MD, stat.writes_await, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_PERC_UTIL_MD, stat.perc_util, server_label, drive_label);
+    }
+
+    metrics
+}
+
+/// Collects drive count metrics (offline, online, total).
+///
+/// Returns a vector of Prometheus metrics for drive counts.
+pub fn collect_drive_count_metrics(stats: &DriveCountStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&DRIVE_OFFLINE_COUNT_MD, stats.offline_count as f64),
+        PrometheusMetric::from_descriptor(&DRIVE_ONLINE_COUNT_MD, stats.online_count as f64),
+        PrometheusMetric::from_descriptor(&DRIVE_COUNT_MD, stats.total_count as f64),
+    ]
+}
+
+/// Process disk I/O statistics.
+///
+/// Contains disk I/O metrics for a specific process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessDiskStats {
+    /// Bytes read from disk
+    pub read_bytes: u64,
+    /// Bytes written to disk
+    pub written_bytes: u64,
+}
+
+/// Collects process disk I/O metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for process disk I/O statistics.
+/// Each metric includes a `direction` label ("read" or "write").
+///
+/// # Arguments
+///
+/// * `stats` - Process disk I/O statistics
+/// * `labels` - Optional additional labels (e.g., process attributes)
+pub fn collect_process_disk_metrics(
+    stats: &ProcessDiskStats,
+    labels: Option<&[(&'static str, Cow<'static, str>)]>,
+) -> Vec<PrometheusMetric> {
+    let mut read_metric = PrometheusMetric::from_descriptor(&PROCESS_DISK_IO_MD, stats.read_bytes as f64);
+    let mut write_metric = PrometheusMetric::from_descriptor(&PROCESS_DISK_IO_MD, stats.written_bytes as f64);
+
+    read_metric.labels.push(("direction", Cow::Borrowed("read")));
+    write_metric.labels.push(("direction", Cow::Borrowed("write")));
+
+    if let Some(l) = labels {
+        read_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        write_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+    }
+
+    vec![read_metric, write_metric]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_drive_detailed_metrics() {
+        let stats = vec![DriveDetailedStats {
+            server: "node1:9000".to_string(),
+            drive: "/data/disk1".to_string(),
+            total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
+            used_bytes: 1024 * 1024 * 1024 * 50,   // 50 GB
+            free_bytes: 1024 * 1024 * 1024 * 50,   // 50 GB
+            used_inodes: 100000,
+            free_inodes: 900000,
+            total_inodes: 1000000,
+            timeout_errors_total: 5,
+            io_errors_total: 10,
+            availability_errors_total: 2,
+            waiting_io: 3,
+            api_latency_micros: 1500,
+            health: 1,
+            reads_per_sec: 100.0,
+            reads_kb_per_sec: 1024.0,
+            reads_await: 5.5,
+            writes_per_sec: 50.0,
+            writes_kb_per_sec: 512.0,
+            writes_await: 10.2,
+            perc_util: 75.5,
+        }];
+
+        let metrics = collect_drive_detailed_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 19);
+
+        // Verify total bytes metric
+        let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name();
+        let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
+        assert!(total_bytes.is_some());
+        assert_eq!(total_bytes.map(|m| m.value), Some(1024.0 * 1024.0 * 1024.0 * 100.0));
+    }
+
+    #[test]
+    fn test_collect_drive_count_metrics() {
+        let stats = DriveCountStats {
+            offline_count: 2,
+            online_count: 8,
+            total_count: 10,
+        };
+
+        let metrics = collect_drive_count_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 3);
+
+        // Verify offline count
+        let offline_name = DRIVE_OFFLINE_COUNT_MD.get_full_metric_name();
+        let offline = metrics.iter().find(|m| m.name == offline_name);
+        assert!(offline.is_some());
+        assert_eq!(offline.map(|m| m.value), Some(2.0));
+    }
+}