fix(sse): optimize is_encrypted for old metadata compatibility (#3113 )

Signed-off-by: w0od <dingboning02@163.com> Co-authored-by: houseme <housemecn@gmail.com>
fix(config): restore default credential startup (#3114 )
2026-08-01 11:02:14 +00:00 · 2026-05-29 11:59:08 +00:00 · 2026-05-29 11:52:46 +00:00 · 2026-05-29 11:20:47 +00:00 · 2026-05-29 09:36:19 +00:00 · 2026-05-29 08:02:50 +00:00
84 changed files with 6044 additions and 1750 deletions
@@ -23,13 +23,15 @@ services:
      - RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
-      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
      - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
      - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
      - RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
+      - RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
+      - RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
    extra_hosts:
      - "host.docker.internal:host-gateway"
@@ -53,13 +55,15 @@ services:
      - RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
-      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
      - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
      - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
      - RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
+      - RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
+      - RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
    extra_hosts:
      - "host.docker.internal:host-gateway"
@@ -83,13 +87,15 @@ services:
      - RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
-      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
      - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
      - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
      - RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
+      - RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
+      - RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
    extra_hosts:
      - "host.docker.internal:host-gateway"
@@ -113,13 +119,15 @@ services:
      - RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
-      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
      - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
      - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
      - RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
+      - RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
+      - RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
    extra_hosts:
      - "host.docker.internal:host-gateway"
@@ -21,8 +21,8 @@ services:
      - RUSTFS_VOLUMES=http://node{0...3}:9000/data/rustfs{0...3}
      - RUSTFS_ADDRESS=0.0.0.0:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_ACCESS_KEY=rustfs-cluster-admin
-      - RUSTFS_SECRET_KEY=rustfs-cluster-secret
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
    platform: linux/amd64
    ports:
      - "9000:9000" # Map port 9001 of the host to port 9000 of the container
@@ -38,8 +38,8 @@ services:
      - RUSTFS_VOLUMES=http://node{0...3}:9000/data/rustfs{0...3}
      - RUSTFS_ADDRESS=0.0.0.0:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_ACCESS_KEY=rustfs-cluster-admin
-      - RUSTFS_SECRET_KEY=rustfs-cluster-secret
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
    platform: linux/amd64
    ports:
      - "9001:9000" # Map port 9002 of the host to port 9000 of the container
@@ -55,8 +55,8 @@ services:
      - RUSTFS_VOLUMES=http://node{0...3}:9000/data/rustfs{0...3}
      - RUSTFS_ADDRESS=0.0.0.0:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_ACCESS_KEY=rustfs-cluster-admin
-      - RUSTFS_SECRET_KEY=rustfs-cluster-secret
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
    platform: linux/amd64
    ports:
      - "9002:9000" # Map port 9003 of the host to port 9000 of the container
@@ -72,8 +72,8 @@ services:
      - RUSTFS_VOLUMES=http://node{0...3}:9000/data/rustfs{0...3}
      - RUSTFS_ADDRESS=0.0.0.0:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_ACCESS_KEY=rustfs-cluster-admin
-      - RUSTFS_SECRET_KEY=rustfs-cluster-secret
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
    platform: linux/amd64
    ports:
      - "9003:9000" # Map port 9004 of the host to port 9000 of the container
@@ -40,8 +40,8 @@ on:

 env:
  # main user
-  S3_ACCESS_KEY: rustfs-ci-admin
-  S3_SECRET_KEY: rustfs-ci-secret
+  S3_ACCESS_KEY: rustfsadmin
+  S3_SECRET_KEY: rustfsadmin
  # alt user (must be different from main for many s3-tests)
  S3_ALT_ACCESS_KEY: rustfsalt
  S3_ALT_SECRET_KEY: rustfsalt
@@ -38,41 +38,65 @@ concurrency:
  cancel-in-progress: true
 permissions:
  contents: read
+  id-token: write

 jobs:
  nix-validation:
    name: Nix Build & Check
    runs-on: ubuntu-latest
    timeout-minutes: 60
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    permissions:
+      contents: read
+      id-token: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Install Nix
-        uses: cachix/install-nix-action@v31
+        uses: DeterminateSystems/determinate-nix-action@v3.21.0
        with:
-          github_access_token: ${{ secrets.GITHUB_TOKEN }}
-          extra_nix_config: |
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          extra-conf: |
            experimental-features = nix-command flakes
+            max-jobs = 1

-      - name: Setup Magic Nix Cache
-        uses: DeterminateSystems/magic-nix-cache-action@v13
+      - name: Cache Nix
+        uses: DeterminateSystems/flakehub-cache-action@v3.21.0

-      - name: Setup Flake Checker
+      - name: Check Nix Flake Inputs
        uses: DeterminateSystems/flake-checker-action@v12
+        with:
+          fail-mode: true
+          ignore-missing-flake-lock: false

      - name: Verify Flake
        run: |
          echo "Checking flake structure and evaluation..."
          nix flake show
-          nix flake check --print-build-logs
+          for attempt in 1 2 3; do
+            if nix flake check --print-build-logs --fallback --option fallback true; then
+              break
+            fi
+            if [ "$attempt" -eq 3 ]; then
+              echo "nix flake check failed after 3 attempts."
+              exit 1
+            fi
+            sleep $((attempt * 15))
+          done

      - name: Build RustFS
        run: |
          echo "Building the default package..."
-          nix build .#default --print-build-logs
+          for attempt in 1 2 3; do
+            if nix build .#default --print-build-logs --fallback --option fallback true; then
+              break
+            fi
+            if [ "$attempt" -eq 3 ]; then
+              echo "nix build failed after 3 attempts."
+              exit 1
+            fi
+            sleep $((attempt * 15))
+          done

      - name: Test Binary
        run: |
@@ -60,7 +60,7 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.95.0"
-version = "1.0.0-beta.4"
+version = "1.0.0-beta.6"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -77,67 +77,67 @@ redundant_clone = "warn"

 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "1.0.0-beta.4" }
-rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.4" }
-rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.4" }
-rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.4" }
-rustfs-common = { path = "crates/common", version = "1.0.0-beta.4" }
-rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.4" }
-rustfs-config = { path = "./crates/config", version = "1.0.0-beta.4" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.4" }
-rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.4" }
-rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.4" }
-rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.4" }
-rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.4" }
-rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.4" }
-rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.4" }
-rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.4" }
-rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.4" }
-rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.4" }
-rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.4" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.4" }
-rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.4" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.4" }
-rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.4" }
-rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.4" }
-rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.4" }
-rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.4" }
-rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.4" }
-rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.4" }
-rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.4" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.4" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.4" }
-rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.4" }
-rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.4" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.4" }
-rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.4" }
-rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.4" }
-rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.4" }
-rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.4" }
+rustfs = { path = "./rustfs", version = "1.0.0-beta.6" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.6" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.6" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.6" }
+rustfs-common = { path = "crates/common", version = "1.0.0-beta.6" }
+rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.6" }
+rustfs-config = { path = "./crates/config", version = "1.0.0-beta.6" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.6" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.6" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.6" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.6" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.6" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.6" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.6" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.6" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.6" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.6" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.6" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.6" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.6" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.6" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.6" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.6" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.6" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.6" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.6" }
+rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.6" }
+rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.6" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.6" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.6" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.6" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.6" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.6" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.6" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.6" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.6" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.6" }

 # Async Runtime and Networking
 async-channel = "2.5.0"
-mysql_async = { version = "0.36.1", default-features = false, features = ["default-rustls", "tracing"], git = "https://github.com/blackbeam/mysql_async", rev = "2bad388283bc3ce48801fc2ffcd22445eb6f3d24" }
+mysql_async = { version = "0.37", default-features = false, features = ["default-rustls", "tracing"] }
 async-compression = { version = "0.4.42" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
-async-nats = "0.48.0"
+async-nats = "0.49.0"
 axum = "0.8.9"
 futures = "0.3.32"
 futures-core = "0.3.32"
 futures-util = "0.3.32"
 pollster = "0.4.0"
-pulsar = { version = "6.7.2", default-features = false, features = ["tokio-rustls-runtime"] }
+pulsar = { version = "6.8.0", default-features = false, features = ["tokio-rustls-runtime"] }
 lapin = { version = "4.10.0", default-features = false, features = ["tokio", "rustls", "rustls--aws_lc_rs"] }
-hyper = { version = "1.9.0", features = ["http2", "http1", "server"] }
+hyper = { version = "1.10.0", features = ["http2", "http1", "server"] }
 hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
 hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
-http = "1.4.0"
+http = "1.4.1"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
-reqwest = { version = "0.13.3", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+reqwest = { version = "0.13.4", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
 rustfs-kafka-async = { version = "1.2.0" }
-socket2 = { version = "0.6.3", features = ["all"] }
+socket2 = { version = "0.6.4", features = ["all"] }
 tokio = { version = "1.52.3", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 tokio-stream = { version = "0.1.18" }
@@ -164,7 +164,7 @@ serde_json = { version = "1.0.150", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"

 # Cryptography and Security
-aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
+aes-gcm = { version = "0.11.0-rc.4", features = ["rand_core"] }
 argon2 = { version = "0.6.0-rc.8" }
 blake2 = "0.11.0-rc.6"
 chacha20poly1305 = { version = "0.11.0-rc.3" }
@@ -185,7 +185,7 @@ zeroize = { version = "1.8.2", features = ["derive"] }
 # Time and Date
 chrono = { version = "0.4.44", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.24", features = ["serde"] }
+jiff = { version = "0.2.28", features = ["serde"] }
 time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }

 # Database
@@ -201,12 +201,12 @@ atoi = "2.0.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.8.17" }
 aws-credential-types = { version = "1.2.14" }
-aws-sdk-s3 = { version = "1.133.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.134.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
 aws-smithy-types = { version = "1.4.8" }
 base64 = "0.22.1"
 base64-simd = "0.8.0"
-brotli = "8.0.2"
+brotli = "8.0.3"
 clap = { version = "4.6.1", features = ["derive", "env"] }
 const-str = { version = "1.1.0", features = ["std", "proc"] }
 convert_case = "0.11.0"
@@ -256,7 +256,7 @@ reed-solomon-erasure = { version = "6.0", default-features = false, features = [
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.12.3" }
 rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] }
-redis = { version = "1.2.1", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
+redis = { version = "1.2.2", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
 rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.2" }
@@ -270,7 +270,7 @@ snafu = "0.9.0"
 snap = "1.1.1"
 starshard = { version = "2.2.0", features = ["rayon", "async", "serde"] }
 strum = { version = "0.28.0", features = ["derive"] }
-sysinfo = "0.39.2"
+sysinfo = "0.39.3"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
 test-case = "3.3.1"
@@ -298,17 +298,17 @@ dial9-tokio-telemetry = "0.3"
 opentelemetry = { version = "0.32.0" }
 opentelemetry-appender-tracing = { version = "0.32.0", features = ["experimental_span_attributes", "experimental_metadata_attributes"] }
 opentelemetry-otlp = { version = "0.32.0", features = ["gzip-http", "reqwest-rustls"] }
-opentelemetry_sdk = { version = "0.32.0", features = ["rt-tokio"] }
+opentelemetry_sdk = { version = "0.32.1", features = ["rt-tokio"] }
 opentelemetry-semantic-conventions = { version = "0.32.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.32.0" }
-pyroscope = { version = "2.0.4", features = ["backend-pprof-rs"] }
+pyroscope = { version = "2.0.5", features = ["backend-pprof-rs"] }

 # FTP and SFTP
 libunftp = { version = "0.23.0", features = ["experimental"] }
 unftp-core = "0.1.0"
 suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.8"
-russh = { version = "0.60.3", git = "https://github.com/Eugeny/russh", rev = "fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" }
+russh = { version = "0.61.1",features = ["serde"] }
 russh-sftp = "2.3.0"

 # WebDAV
@@ -105,6 +105,8 @@ RUN groupadd -g 10001 rustfs && \

 ENV RUSTFS_ADDRESS=":9000" \
    RUSTFS_CONSOLE_ADDRESS=":9001" \
+    RUSTFS_ACCESS_KEY="rustfsadmin" \
+    RUSTFS_SECRET_KEY="rustfsadmin" \
    RUSTFS_CONSOLE_ENABLE="true" \
    RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS="*" \
    RUSTFS_VOLUMES="/data" \
@@ -7,6 +7,7 @@
  <a href="https://github.com/rustfs/rustfs/actions/workflows/docker.yml"><img alt="Build and Push Docker Images" src="https://github.com/rustfs/rustfs/actions/workflows/docker.yml/badge.svg" /></a>
  <img alt="GitHub commit activity" src="https://img.shields.io/github/commit-activity/m/rustfs/rustfs"/>
  <img alt="Github Last Commit" src="https://img.shields.io/github/last-commit/rustfs/rustfs"/>
+  <a href="https://discord.gg/NcKBCEJp6P"><img alt="Discord" src="https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white" /></a>
  <a href="https://hellogithub.com/repository/rustfs/rustfs" target="_blank"><img src="https://abroad.hellogithub.com/v1/widgets/recommend.svg?rid=b95bcb72bdc340b68f16fdf6790b7d5b&claim_uid=MsbvjYeLDKAH457&theme=small" alt="Featured｜HelloGitHub" /></a>
 </p>

@@ -115,7 +116,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest

 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.4
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.6
 ```

 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
@@ -295,6 +296,7 @@ If you have any questions or need assistance:
 - [Documentation](https://docs.rustfs.com) - The manual you should read
 - [Changelog](https://github.com/rustfs/rustfs/releases) - What we broke and fixed
 - [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Where the community lives
+- [Discord](https://discord.gg/NcKBCEJp6P) - Chat with the RustFS community

 ## Contact

@@ -113,7 +113,7 @@ RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest

 # 使用指定版本运行
- docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.4
+ docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.6
 ```

 您也可以使用 Docker Compose。使用根目录下的 `docker-compose.yml` 文件：
@@ -71,6 +71,24 @@ Current guidance:
 - `RUSTFS_SCANNER_IDLE_MODE` (canonical)
 - `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` (canonical)

+## Health compatibility switches
+
+- `RUSTFS_HEALTH_ENDPOINT_ENABLE`
+  - controls canonical `/health`, `/health/live`, and `/health/ready` endpoint exposure.
+- `RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE`
+  - enables minimal payload mode for GET health responses (`status`, `ready` only).
+- `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS`
+  - TTL for readiness cache evaluation.
+- `RUSTFS_HEALTH_COMPAT_BUSY_CHECK_ENABLE`
+  - enables busy protection behavior for health probes.
+  - default is `false`.
+- `RUSTFS_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS`
+  - max active HTTP requests; health probes return `429` when active requests reach or exceed this value.
+  - `0` disables thresholding even if busy protection is enabled.
+- `RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE`
+  - enables KMS readiness enforcement for `/health/ready`.
+  - default is `false`.
+
 ## Drive timeout environment variables

 - `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
@@ -155,12 +155,6 @@ pub const ENV_RUSTFS_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
 /// Environment variable for server secret key file.
 pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE";

-/// Environment variable to explicitly allow public default root credentials.
-///
-/// This is intended for local development only. Production startup paths should
-/// provide non-default `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` values.
-pub const ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS: &str = "RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS";
-
 /// Environment variable controlling startup behavior when unsupported filesystem types are detected.
 ///
 /// Accepted values:
@@ -26,3 +26,20 @@ pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
 /// When enabled, only `status` and `ready` fields are returned.
 pub const ENV_HEALTH_MINIMAL_RESPONSE_ENABLE: &str = "RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE";
 pub const DEFAULT_HEALTH_MINIMAL_RESPONSE_ENABLE: bool = false;
+
+/// Enable busy protection for health probes.
+/// When enabled with a positive request threshold, alias health probes may
+/// return 429 when active HTTP requests exceed the threshold.
+pub const ENV_HEALTH_COMPAT_BUSY_CHECK_ENABLE: &str = "RUSTFS_HEALTH_COMPAT_BUSY_CHECK_ENABLE";
+pub const DEFAULT_HEALTH_COMPAT_BUSY_CHECK_ENABLE: bool = false;
+
+/// Max active HTTP requests; alias health probes report busy (429) when active requests reach or exceed this value.
+/// Set to 0 to disable thresholding even when busy protection is enabled.
+pub const ENV_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS: &str = "RUSTFS_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS";
+pub const DEFAULT_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS: usize = 0;
+
+/// Enable KMS readiness check for alias readiness probes.
+/// When enabled, `/health/ready` additionally requires KMS service to be
+/// in running state if a global KMS manager exists.
+pub const ENV_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE: &str = "RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE";
+pub const DEFAULT_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE: bool = false;
@@ -191,7 +191,7 @@ RUST_LOG=debug cargo test test_local_kms_end_to_end -- --nocapture
 4. **Monitor ports**
   ```bash
   netstat -an | grep 9050
-   curl http://127.0.0.1:9050/minio/health/ready
+   curl http://127.0.0.1:9050/health/ready
   ```

 ## 📊 Coverage
@@ -63,6 +63,10 @@ mod archive_download_integrity_test;
 #[cfg(test)]
 mod list_objects_v2_pagination_test;

+// Regression test for Issue #3107: mc mirror small-bucket listing must not time out.
+#[cfg(test)]
+mod mc_mirror_small_bucket_test;
+
 // Policy variables tests
 #[cfg(test)]
 mod policy;
@@ -142,4 +146,7 @@ mod replication_extension_test;
 #[cfg(test)]
 mod snowball_auto_extract_test;

+#[cfg(test)]
+mod namespace_lock_quorum_test;
+
 pub mod tls_gen;
@@ -0,0 +1,124 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, RustFSTestEnvironment};
+use serial_test::serial;
+use std::path::Path;
+use std::process::Command;
+use std::time::Duration;
+use tokio::fs;
+use tokio::time::timeout;
+use tracing::info;
+
+const BUCKET: &str = "ddd";
+const OBJECT_COUNT: usize = 484;
+
+type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+async fn create_issue_3107_fixture(root: &Path) -> TestResult {
+    for idx in 0..OBJECT_COUNT {
+        let object_path = root.join("ddd").join("requirements").join(format!("file-{idx:04}.txt"));
+        fs::create_dir_all(object_path.parent().expect("object parent should exist")).await?;
+        fs::write(object_path, format!("issue-3107 payload {idx}\n").repeat(128)).await?;
+    }
+
+    for dir in ["aaa", "ccc", "ccc-package"] {
+        let object_path = root.join(dir).join("marker.txt");
+        fs::create_dir_all(object_path.parent().expect("object parent should exist")).await?;
+        fs::write(object_path, dir.as_bytes()).await?;
+    }
+
+    Ok(())
+}
+
+fn mc_available() -> bool {
+    Command::new("mc")
+        .arg("--version")
+        .output()
+        .is_ok_and(|output| output.status.success())
+}
+
+fn run_mc(args: &[&str]) -> TestResult {
+    let output = Command::new("mc").args(args).output()?;
+    if !output.status.success() {
+        return Err(format!(
+            "mc {:?} failed: status={:?}, stdout={}, stderr={}",
+            args,
+            output.status.code(),
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+fn count_files(root: &Path) -> usize {
+    walkdir::WalkDir::new(root)
+        .into_iter()
+        .filter_map(Result::ok)
+        .filter(|entry| entry.file_type().is_file())
+        .count()
+}
+
+#[tokio::test]
+#[serial]
+async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult {
+    crate::common::init_logging();
+    info!("Starting issue #3107 mc mirror regression test");
+    if !mc_available() {
+        info!("Skipping issue #3107 mc mirror regression test because mc is not installed");
+        return Ok(());
+    }
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(BUCKET).await?;
+
+    let alias = format!("rustfs-issue-3107-{}", uuid::Uuid::new_v4());
+    run_mc(&["alias", "set", &alias, &env.url, DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY])?;
+
+    let fixture = Path::new(&env.temp_dir).join("issue-3107-fixture");
+    let backup = Path::new(&env.temp_dir).join("issue-3107-backup");
+    fs::create_dir_all(&fixture).await?;
+    fs::create_dir_all(&backup).await?;
+    create_issue_3107_fixture(&fixture).await?;
+
+    let bucket_target = format!("{alias}/{BUCKET}");
+    let source_path = fixture.to_string_lossy().to_string();
+    run_mc(&["mirror", "--overwrite", &source_path, &bucket_target])?;
+
+    let backup_path = backup.join("ddd-backup");
+    let backup_path_string = backup_path.to_string_lossy().to_string();
+    timeout(
+        Duration::from_secs(20),
+        tokio::task::spawn_blocking({
+            let bucket_target = bucket_target.clone();
+            let backup_path_string = backup_path_string.clone();
+            move || run_mc(&["mirror", "--overwrite", &bucket_target, &backup_path_string])
+        }),
+    )
+    .await
+    .map_err(|_| "mc mirror timed out while listing/comparing a small bucket")???;
+
+    let mirrored_count = count_files(&backup_path);
+    assert_eq!(
+        mirrored_count,
+        OBJECT_COUNT + 3,
+        "mc mirror should copy every object from the issue #3107 small bucket fixture"
+    );
+
+    run_mc(&["alias", "remove", &alias])?;
+    Ok(())
+}
@@ -0,0 +1,122 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::RustFSTestClusterEnvironment;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::SdkError;
+use bytes::Bytes;
+use serial_test::serial;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+use tracing::{info, warn};
+
+const BUCKET: &str = "namespace-lock-quorum-bucket";
+const KEY: &str = "thumb/79/concurrent-overwrite.jpg";
+
+type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+async fn put_object(client: Client, payload: Vec<u8>, writer_id: usize) -> Result<(), String> {
+    client
+        .put_object()
+        .bucket(BUCKET)
+        .key(KEY)
+        .body(Bytes::from(payload).into())
+        .send()
+        .await
+        .map(|_| ())
+        .map_err(|err| format_s3_error(err, writer_id))
+}
+
+fn format_s3_error(err: SdkError<aws_sdk_s3::operation::put_object::PutObjectError>, writer_id: usize) -> String {
+    match err {
+        SdkError::ServiceError(service_err) => {
+            let err = service_err.err();
+            let code = err.meta().code().unwrap_or("<unknown>");
+            let message = err.meta().message().unwrap_or("<empty>");
+            format!("writer {writer_id} returned service error {code}: {message}")
+        }
+        other => format!("writer {writer_id} returned SDK error: {other:?}"),
+    }
+}
+
+#[tokio::test]
+#[serial]
+async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum() -> TestResult {
+    crate::common::init_logging();
+    info!("Starting namespace lock quorum regression test with auto cluster");
+
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    // Keep the regression focused on false quorum-loss errors, not ordinary lock
+    // wait exhaustion under a heavily contended same-key overwrite workload.
+    cluster.set_env("RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT", "20");
+    cluster.start().await?;
+    cluster.create_test_bucket(BUCKET).await?;
+
+    let clients = cluster.create_all_clients()?;
+    let first_payload = b"initial object contents".to_vec();
+    put_object(clients[0].clone(), first_payload, 0).await?;
+
+    let writer_count = clients.len() * 2;
+    let barrier = Arc::new(Barrier::new(writer_count));
+    let mut handles = Vec::with_capacity(writer_count);
+
+    for writer_id in 0..writer_count {
+        let client = clients[writer_id % clients.len()].clone();
+        let barrier = barrier.clone();
+        let payload = format!("replacement payload from writer {writer_id:02}").into_bytes();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            put_object(client, payload, writer_id).await
+        }));
+    }
+
+    let mut failures = Vec::new();
+    for handle in handles {
+        match handle.await {
+            Ok(Ok(())) => {}
+            Ok(Err(err)) => failures.push(err),
+            Err(err) => failures.push(format!("writer task join failed: {err}")),
+        }
+    }
+
+    if !failures.is_empty() {
+        for failure in &failures {
+            warn!("concurrent overwrite failure: {}", failure);
+        }
+    }
+
+    assert!(
+        failures.is_empty(),
+        "concurrent overwrites must not surface namespace lock quorum failures: {failures:#?}"
+    );
+
+    let body = clients[0]
+        .get_object()
+        .bucket(BUCKET)
+        .key(KEY)
+        .send()
+        .await?
+        .body
+        .collect()
+        .await?
+        .into_bytes();
+    let body = std::str::from_utf8(&body)?;
+    assert!(
+        body.starts_with("replacement payload from writer "),
+        "final object body should be one of the successful overwrite payloads, got {body:?}"
+    );
+
+    clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?;
+    Ok(())
+}
@@ -70,6 +70,36 @@ mod tests {
        Ok(builder.into_inner().await?.into_inner())
    }

+    fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
+        let path = format!("../{victim_bucket}/evil-injected.txt");
+        let data = b"injected-body";
+        let mut header = [0u8; 512];
+
+        header[..path.len()].copy_from_slice(path.as_bytes());
+        header[100..108].copy_from_slice(b"0000644\0");
+        header[108..116].copy_from_slice(b"0000000\0");
+        header[116..124].copy_from_slice(b"0000000\0");
+        let size = format!("{:011o}\0", data.len());
+        header[124..136].copy_from_slice(size.as_bytes());
+        header[136..148].copy_from_slice(b"00000000000\0");
+        header[148..156].fill(b' ');
+        header[156] = b'0';
+        header[257..263].copy_from_slice(b"ustar\0");
+        header[263..265].copy_from_slice(b"00");
+
+        let checksum: u32 = header.iter().map(|byte| *byte as u32).sum();
+        let checksum = format!("{:06o}\0 ", checksum);
+        header[148..156].copy_from_slice(checksum.as_bytes());
+
+        let mut archive = Vec::new();
+        archive.extend_from_slice(&header);
+        archive.extend_from_slice(data);
+        let padding = (512 - (data.len() % 512)) % 512;
+        archive.extend(std::iter::repeat_n(0, padding));
+        archive.extend_from_slice(&[0u8; 1024]);
+        archive
+    }
+
    #[tokio::test]
    #[serial]
    async fn snowball_auto_extract_supports_minio_prefix_and_directory_markers() -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -273,6 +303,49 @@ mod tests {
        Ok(())
    }

+    #[tokio::test]
+    #[serial]
+    async fn snowball_auto_extract_rejects_parent_dir_entry_without_cross_bucket_write()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let attacker_bucket = "snowball-traversal-source";
+        let victim_bucket = "snowball-traversal-victim";
+        let archive = build_archive_with_parent_dir_entry(victim_bucket);
+
+        client.create_bucket().bucket(attacker_bucket).send().await?;
+        client.create_bucket().bucket(victim_bucket).send().await?;
+
+        let err = client
+            .put_object()
+            .bucket(attacker_bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .body(ByteStream::from(archive))
+            .send()
+            .await
+            .expect_err("parent directory archive entry should be rejected");
+        let service_err = err.into_service_error();
+        assert_eq!(service_err.code(), Some("InvalidArgument"));
+
+        let victim_err = client
+            .head_object()
+            .bucket(victim_bucket)
+            .key("evil-injected.txt")
+            .send()
+            .await
+            .expect_err("rejected archive entry must not write into the victim bucket");
+        let victim_service_err = victim_err.into_service_error();
+        assert_eq!(victim_service_err.code(), Some("NotFound"));
+
+        env.stop_server();
+        Ok(())
+    }
+
    #[tokio::test]
    #[serial]
    async fn snowball_auto_extract_prefers_exact_minio_prefix_over_suffix_fallback() -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -131,7 +131,7 @@ aws-smithy-http-client.workspace = true
 metrics = { workspace = true }

 [dev-dependencies]
-tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] }
 criterion = { workspace = true, features = ["html_reports"] }
 temp-env = { workspace = true, features = ["async_closure"] }
 tracing-subscriber = { workspace = true }
@@ -274,10 +274,10 @@ mod tests {
        assert!(successes.len() >= 2);
    }

-    #[tokio::test]
+    #[tokio::test(start_paused = true)]
    async fn test_batch_processor_quorum_returns_before_slow_tail() {
        let processor = AsyncBatchProcessor::new(4);
-        let started = std::time::Instant::now();
+        let started = tokio::time::Instant::now();

        let tasks: Vec<_> = [(10_u64, Ok(1_i32)), (15, Ok(2)), (250, Ok(3))]
            .into_iter()
@@ -295,10 +295,10 @@ mod tests {
        assert!(started.elapsed() < Duration::from_millis(100));
    }

-    #[tokio::test]
+    #[tokio::test(start_paused = true)]
    async fn test_batch_processor_quorum_fails_once_quorum_becomes_impossible() {
        let processor = AsyncBatchProcessor::new(4);
-        let started = std::time::Instant::now();
+        let started = tokio::time::Instant::now();

        let tasks: Vec<_> = vec![
            (10_u64, Ok(1_i32)),
@@ -48,8 +48,8 @@ use rustfs_common::heal_channel::rep_has_active_rules;
 use rustfs_common::metrics::{IlmAction, Metrics};
 use rustfs_config::{
    DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_QUEUE_SEND_TIMEOUT_MS, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
-    DEFAULT_TRANSITION_WORKERS_CAP, ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, ENV_TRANSITION_QUEUE_CAPACITY,
-    ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
+    DEFAULT_TRANSITION_WORKERS_CAP, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS,
+    ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
 };
 use rustfs_data_usage::TierStats;
 use rustfs_filemeta::{
@@ -75,6 +75,8 @@ use time::OffsetDateTime;
 use tokio::select;
 use tokio::sync::mpsc::{Receiver, Sender};
 use tokio::sync::{RwLock, mpsc};
+use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
 use uuid::Uuid;
 use xxhash_rust::xxh64;
@@ -145,7 +147,7 @@ fn is_immediate_transition_source(src: &LcEventSrc) -> bool {

 #[cfg(any(test, debug_assertions))]
 fn should_force_immediate_transition_enqueue_timeout() -> bool {
-    env::var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT)
+    env::var(rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT)
        .ok()
        .is_some_and(|value| value == "1")
 }
@@ -587,12 +589,16 @@ impl ExpiryOp for TransitionTask {
    }
 }

+struct TransitionWorker {
+    cancel: CancellationToken,
+    handle: JoinHandle<()>,
+}
+
 pub struct TransitionState {
    transition_tx: A_Sender<Option<TransitionTask>>,
    transition_rx: A_Receiver<Option<TransitionTask>>,
    pub num_workers: AtomicI64,
-    kill_tx: A_Sender<()>,
-    kill_rx: A_Receiver<()>,
+    workers: Mutex<Vec<TransitionWorker>>,
    transition_queue_capacity: usize,
    transition_queue_send_timeout: StdDuration,
    active_tasks: AtomicI64,
@@ -621,13 +627,11 @@ impl TransitionState {
        let capacity = capacity.max(1);
        let queue_send_timeout = resolve_transition_queue_send_timeout();
        let (tx1, rx1) = bounded(capacity);
-        let (tx2, rx2) = bounded(1);
        Arc::new(Self {
            transition_tx: tx1,
            transition_rx: rx1,
            num_workers: AtomicI64::new(0),
-            kill_tx: tx2,
-            kill_rx: rx2,
+            workers: Mutex::new(Vec::new()),
            transition_queue_capacity: capacity,
            transition_queue_send_timeout: queue_send_timeout,
            active_tasks: AtomicI64::new(0),
@@ -849,10 +853,12 @@ impl TransitionState {
        Self::counter_value(&self.compensation_running_tasks)
    }

-    pub async fn worker(api: Arc<ECStore>) {
+    async fn worker_with_cancel(api: Arc<ECStore>, cancel_token: CancellationToken) {
        loop {
            select! {
-                _ = GLOBAL_TransitionState.kill_rx.recv() => {
+                biased;
+
+                _ = cancel_token.cancelled() => {
                    return;
                }
                task = GLOBAL_TransitionState.transition_rx.recv() => {
@@ -952,33 +958,45 @@ impl TransitionState {
        }
        // Allow environment override of maximum workers
        let absolute_max = resolve_transition_workers_absolute_max();
-        n = std::cmp::min(n, absolute_max);
+        n = n.clamp(0, absolute_max);

-        let previous_num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
-        let mut num_workers = previous_num_workers;
-        while num_workers < n {
+        Self::resize_workers_to(api, n, requested, absolute_max);
+    }
+
+    fn resize_workers_to(api: Arc<ECStore>, n: i64, requested: i64, absolute_max: i64) {
+        let target = n as usize;
+        let mut workers = GLOBAL_TransitionState.workers.lock().unwrap();
+        let tracked_workers = workers.len();
+        workers.retain(|worker| !worker.handle.is_finished());
+        let pruned_finished_workers = tracked_workers.saturating_sub(workers.len());
+        let previous_num_workers = workers.len() as i64;
+
+        while workers.len() < target {
            let clone_api = api.clone();
-            tokio::spawn(async move {
-                TransitionState::worker(clone_api).await;
+            let cancel = CancellationToken::new();
+            let worker_cancel = cancel.clone();
+            let handle = tokio::spawn(async move {
+                TransitionState::worker_with_cancel(clone_api, worker_cancel).await;
            });
-            num_workers += 1;
-            GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst);
+            workers.push(TransitionWorker { cancel, handle });
        }

-        let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
-        while num_workers > n {
-            let worker = GLOBAL_TransitionState.kill_tx.clone();
-            let _ = worker.send(()).await;
-            num_workers -= 1;
-            GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst);
+        while workers.len() > target {
+            if let Some(worker) = workers.pop() {
+                worker.cancel.cancel();
+            }
        }

+        let current_workers = workers.len() as i64;
+        GLOBAL_TransitionState.num_workers.store(current_workers, Ordering::SeqCst);
+
        info!(
            requested_transition_workers = requested,
            effective_transition_workers = n,
            absolute_max_workers = absolute_max,
            previous_transition_workers = previous_num_workers,
-            current_transition_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst),
+            current_transition_workers = current_workers,
+            pruned_finished_transition_workers = pruned_finished_workers,
            "transition workers updated"
        );
    }
@@ -2264,7 +2282,8 @@ mod tests {
        lifecycle_rule_has_date_expiration, lifecycle_version_purge_state_from_completed_targets,
        mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, replication_state_for_delete,
        resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count,
-        should_defer_date_expiry_for_recent_config_update, should_reuse_lifecycle_delete_replication_state,
+        resolve_transition_workers_absolute_max, should_defer_date_expiry_for_recent_config_update,
+        should_reuse_lifecycle_delete_replication_state,
    };
    use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
    use crate::bucket::metadata_sys;
@@ -2655,15 +2674,45 @@ mod tests {
        })
        .await;

-        let current_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
-        if original_workers > 0 {
-            TransitionState::update_workers(ecstore, original_workers).await;
-        } else {
-            for _ in 0..current_workers {
-                let _ = GLOBAL_TransitionState.kill_tx.send(()).await;
-                GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst);
-            }
-        }
+        let absolute_max = resolve_transition_workers_absolute_max();
+        TransitionState::resize_workers_to(ecstore, original_workers, original_workers, absolute_max);
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn transition_worker_resize_cancels_removed_workers_directly() {
+        let (_paths, ecstore) = setup_test_env().await;
+        let original_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
+        let absolute_max = resolve_transition_workers_absolute_max();
+
+        TransitionState::resize_workers_to(ecstore.clone(), 0, 0, absolute_max);
+        assert_eq!(GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst), 0);
+
+        TransitionState::resize_workers_to(ecstore.clone(), 2, 2, absolute_max);
+        let worker_tokens = {
+            let workers = GLOBAL_TransitionState.workers.lock().unwrap();
+            assert_eq!(workers.len(), 2);
+            workers.iter().map(|worker| worker.cancel.clone()).collect::<Vec<_>>()
+        };
+
+        TransitionState::resize_workers_to(ecstore.clone(), 1, 1, absolute_max);
+
+        assert_eq!(GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst), 1);
+        assert_eq!(worker_tokens.iter().filter(|token| token.is_cancelled()).count(), 1);
+
+        let remaining_token = {
+            let workers = GLOBAL_TransitionState.workers.lock().unwrap();
+            assert_eq!(workers.len(), 1);
+            let token = workers[0].cancel.clone();
+            assert!(!token.is_cancelled());
+            token
+        };
+
+        TransitionState::resize_workers_to(ecstore.clone(), 0, 0, absolute_max);
+        assert_eq!(GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst), 0);
+        assert!(remaining_token.is_cancelled());
+
+        TransitionState::resize_workers_to(ecstore, original_workers, original_workers, absolute_max);
    }

    #[test]
@@ -3138,6 +3187,76 @@ mod tests {
        assert!(!parts.user_defined.contains_key(RUSTFS_MULTIPART_OBJECT_KEY));
    }

+    #[tokio::test]
+    #[serial]
+    async fn repeated_upload_part_overwrites_previous_part_state() {
+        let (_paths, ecstore) = setup_test_env().await;
+        let bucket = format!("multipart-overwrite-{}", Uuid::new_v4().simple());
+        let object = "overwrite/object.txt";
+        create_test_bucket(&ecstore, &bucket).await;
+
+        let upload = ecstore
+            .new_multipart_upload(&bucket, object, &ObjectOptions::default())
+            .await
+            .expect("multipart upload should be created");
+
+        let mut first = PutObjReader::from_vec(vec![1, 2, 3]);
+        let first_part = ecstore
+            .put_object_part(&bucket, object, &upload.upload_id, 1, &mut first, &ObjectOptions::default())
+            .await
+            .expect("first multipart part should be uploaded");
+
+        let mut second = PutObjReader::from_vec(vec![4, 5, 6, 7]);
+        let second_part = ecstore
+            .put_object_part(&bucket, object, &upload.upload_id, 1, &mut second, &ObjectOptions::default())
+            .await
+            .expect("second multipart part should overwrite the previous part");
+
+        assert_ne!(
+            first_part.etag, second_part.etag,
+            "the overwrite path should persist the latest part metadata rather than reusing stale state"
+        );
+
+        let parts = ecstore
+            .list_object_parts(
+                &bucket,
+                object,
+                &upload.upload_id,
+                None,
+                crate::set_disk::MAX_PARTS_COUNT,
+                &ObjectOptions::default(),
+            )
+            .await
+            .expect("multipart parts should be readable after overwrite");
+
+        assert_eq!(parts.parts.len(), 1, "only the latest version of part 1 should remain visible");
+        assert_eq!(parts.parts[0].part_num, 1);
+        assert_eq!(parts.parts[0].etag, second_part.etag);
+        assert_eq!(parts.parts[0].size, second_part.size);
+        assert_eq!(parts.parts[0].actual_size, second_part.actual_size);
+
+        let completed = ecstore
+            .complete_multipart_upload(
+                &bucket,
+                object,
+                &upload.upload_id,
+                vec![crate::store_api::CompletePart {
+                    part_num: 1,
+                    etag: second_part.etag.clone(),
+                    checksum_crc32: None,
+                    checksum_crc32c: None,
+                    checksum_sha1: None,
+                    checksum_sha256: None,
+                    checksum_crc64nvme: None,
+                }],
+                &ObjectOptions::default(),
+            )
+            .await
+            .expect("complete multipart upload should succeed with the latest overwritten part");
+
+        assert_eq!(completed.size, second_part.size as i64);
+    }
+
    #[tokio::test]
    #[serial]
    async fn cleanup_removes_empty_multipart_sha_dirs() {
@@ -442,10 +442,13 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                {
                    finished_fn(&errs).await;
                }
-                if errs.iter().flatten().any(|err| *err == DiskError::Timeout) {
-                    return Err(DiskError::Timeout);
-                }
-
+                // All remaining readers are either at EOF or failed. Earlier logic
+                // returned Timeout here for even a single stalled drive, despite
+                // `has_err` being within the tolerated drive-failure budget. That
+                // makes small distributed listings fail once healthy quorum readers
+                // reach EOF but one remote walk stream is slow/stalled. Only the
+                // `has_err > opts.disks.len() - opts.min_disks` branch above should
+                // turn tolerated reader failures into request failures.
                // error!("list_path_raw: at_eof + has_err == readers.len() break {:?}", &errs);
                break;
            }
@@ -486,13 +489,23 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
        return Err(err);
    }

+    // The merge consumer can finish successfully before every producer finishes
+    // (for example after reaching EOF quorum while a tolerated drive is stalled,
+    // or after the requested listing limit is satisfied). Cancel remaining walk
+    // jobs before joining them so list calls do not wait for slow remote streams.
+    cancel_rx.cancel();
+
    let results = join_all(jobs).await;
    let mut job_errs = Vec::new();
    for result in results {
        match result {
            Ok(Ok(())) => {}
            Ok(Err(err)) => {
-                error!("list_path_raw producer err {:?}", err);
+                if matches!(err, DiskError::FileNotFound | DiskError::VolumeNotFound) {
+                    warn!("list_path_raw producer missing path {:?}", err);
+                } else {
+                    error!("list_path_raw producer err {:?}", err);
+                }
                job_errs.push(err);
            }
            Err(err) => {
@@ -541,18 +554,34 @@ mod tests {
            CancellationToken::new(),
            ListPathRawOptions {
                disks: vec![None, None],
-                min_disks: 1,
+                min_disks: 2,
                test_reader_behaviors: vec![TestReaderBehavior::Stall, TestReaderBehavior::Eof],
                peek_timeout: Some(Duration::from_millis(20)),
                ..Default::default()
            },
        )
        .await
-        .expect_err("stalled reader should make listing fail explicitly");
+        .expect_err("stalled reader should fail when read quorum cannot be met");

        assert_eq!(err, DiskError::Timeout);
    }

+    #[tokio::test]
+    async fn list_path_raw_tolerates_stalled_reader_after_quorum_eof() {
+        list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![None, None, None],
+                min_disks: 2,
+                test_reader_behaviors: vec![TestReaderBehavior::Eof, TestReaderBehavior::Eof, TestReaderBehavior::Stall],
+                peek_timeout: Some(Duration::from_millis(20)),
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("listing should complete when healthy quorum reached EOF and only a tolerated drive stalled");
+    }
+
    #[tokio::test]
    async fn list_path_raw_returns_timeout_when_producer_fails_after_partial_entry() {
        let seen = Arc::new(Mutex::new(Vec::new()));
@@ -57,14 +57,15 @@ use std::{
 use time::OffsetDateTime;
 use tokio::fs::{self, File};
 use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind};
-use tokio::sync::RwLock;
-use tokio::time::interval;
+use tokio::sync::{Notify, RwLock};
+use tokio::time::{Instant, interval_at, timeout};
 use tracing::{debug, error, info, warn};
 use uuid::Uuid;

 const DELETED_OBJECTS_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 5);
 const STALE_TMP_OBJECT_EXPIRY: Duration = Duration::from_secs(24 * 60 * 60);
 const RUSTFS_META_TMP_OLD_BUCKET: &str = ".rustfs.sys/tmp-old";
+const STARTUP_CLEANUP_WAIT_TIMEOUT: Duration = Duration::from_secs(2);

 #[derive(Debug, Clone)]
 pub struct FormatInfo {
@@ -331,6 +332,8 @@ pub struct LocalDisk {
    // pub format_data: Mutex<Vec<u8>>,
    // pub format_file_info: Mutex<Option<Metadata>>,
    // pub format_last_check: Mutex<Option<OffsetDateTime>>,
+    startup_cleanup_ready: Arc<AtomicU32>,
+    startup_cleanup_notify: Arc<Notify>,
    exit_signal: Option<tokio::sync::broadcast::Sender<()>>,
 }

@@ -370,7 +373,15 @@ impl LocalDisk {

        ensure_data_usage_layout(&root).await.map_err(DiskError::from)?;

-        if cleanup && let Err(err) = Self::cleanup_tmp_on_startup(&root).await {
+        let startup_cleanup_ready = Arc::new(AtomicU32::new(u32::from(!cleanup)));
+        let startup_cleanup_notify = Arc::new(Notify::new());
+
+        if cleanup
+            && let Err(err) =
+                Self::cleanup_tmp_on_startup(&root, startup_cleanup_ready.clone(), startup_cleanup_notify.clone()).await
+        {
+            startup_cleanup_ready.store(1, Ordering::Release);
+            startup_cleanup_notify.notify_waiters();
            warn!(root = ?root, error = ?err, "failed to cleanup temporary data during disk startup");
        }

@@ -466,6 +477,8 @@ impl LocalDisk {
            // format_last_check: Mutex::new(format_last_check),
            path_cache: Arc::new(ParkingLotRwLock::new(HashMap::with_capacity(2048))),
            current_dir: Arc::new(OnceLock::new()),
+            startup_cleanup_ready,
+            startup_cleanup_notify,
            exit_signal: None,
        };
        let (info, _root) = get_disk_info(root).await?;
@@ -497,7 +510,8 @@ impl LocalDisk {
    }

    async fn cleanup_deleted_objects_loop(root: PathBuf, mut exit_rx: tokio::sync::broadcast::Receiver<()>) {
-        let mut interval = interval(DELETED_OBJECTS_CLEANUP_INTERVAL);
+        let start_at = Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL;
+        let mut interval = interval_at(start_at, DELETED_OBJECTS_CLEANUP_INTERVAL);
        loop {
            tokio::select! {
                _ = interval.tick() => {
@@ -525,12 +539,18 @@ impl LocalDisk {
        root.join(meta_path)
    }

-    async fn cleanup_tmp_on_startup(root: &Path) -> Result<()> {
+    async fn cleanup_tmp_on_startup(
+        root: &Path,
+        startup_cleanup_ready: Arc<AtomicU32>,
+        startup_cleanup_notify: Arc<Notify>,
+    ) -> Result<()> {
        let tmp_path = Self::meta_path(root, RUSTFS_META_TMP_BUCKET);
        let tmp_old_path = Self::meta_path(root, RUSTFS_META_TMP_OLD_BUCKET).join(Uuid::new_v4().to_string());

        rename_all(&tmp_path, &tmp_old_path, root).await?;

+        tokio::fs::create_dir_all(Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET)).await?;
+
        let tmp_old_root = Self::meta_path(root, RUSTFS_META_TMP_OLD_BUCKET);
        tokio::spawn(async move {
            if let Err(err) = tokio::fs::remove_dir_all(&tmp_old_root).await
@@ -538,12 +558,35 @@ impl LocalDisk {
            {
                warn!(path = ?tmp_old_root, error = ?err, "failed to remove old temporary data");
            }
+            startup_cleanup_ready.store(1, Ordering::Release);
+            startup_cleanup_notify.notify_waiters();
        });

-        tokio::fs::create_dir_all(Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET)).await?;
        Ok(())
    }

+    async fn wait_for_startup_cleanup(&self) {
+        if self.startup_cleanup_ready.load(Ordering::Acquire) != 0 {
+            return;
+        }
+
+        if wait_for_startup_cleanup_signal(
+            self.startup_cleanup_ready.as_ref(),
+            self.startup_cleanup_notify.as_ref(),
+            STARTUP_CLEANUP_WAIT_TIMEOUT,
+        )
+        .await
+        {
+            debug!(disk = %self.endpoint, "startup cleanup barrier released before walk_dir");
+        } else {
+            warn!(
+                disk = %self.endpoint,
+                timeout_ms = STARTUP_CLEANUP_WAIT_TIMEOUT.as_millis(),
+                "startup cleanup barrier timed out; continuing walk_dir"
+            );
+        }
+    }
+
    async fn cleanup_stale_tmp_objects(root: PathBuf) -> Result<()> {
        Self::cleanup_stale_tmp_objects_with_expiry(root, STALE_TMP_OBJECT_EXPIRY).await
    }
@@ -2375,6 +2418,8 @@ impl DiskAPI for LocalDisk {
    // FIXME: TODO: io.writer TODO cancel
    #[tracing::instrument(level = "debug", skip(self, wr))]
    async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
+        self.wait_for_startup_cleanup().await;
+
        let volume_dir = self.get_bucket_path(&opts.bucket)?;

        if !skip_access_checks(&opts.bucket)
@@ -3105,6 +3150,31 @@ impl DiskAPI for LocalDisk {
    }
 }

+async fn wait_for_startup_cleanup_signal(
+    startup_cleanup_ready: &AtomicU32,
+    startup_cleanup_notify: &Notify,
+    wait_timeout: Duration,
+) -> bool {
+    if startup_cleanup_ready.load(Ordering::Acquire) != 0 {
+        return true;
+    }
+
+    timeout(wait_timeout, async {
+        loop {
+            if startup_cleanup_ready.load(Ordering::Acquire) != 0 {
+                return;
+            }
+            let notified = startup_cleanup_notify.notified();
+            if startup_cleanup_ready.load(Ordering::Acquire) != 0 {
+                return;
+            }
+            notified.await;
+        }
+    })
+    .await
+    .is_ok()
+}
+
 #[tracing::instrument]
 async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInfo, bool)> {
    let drive_path = drive_path.to_string_lossy().to_string();
@@ -3157,7 +3227,9 @@ mod test {
        fs::create_dir_all(leftover.parent().unwrap()).await.unwrap();
        fs::write(&leftover, b"temporary").await.unwrap();

-        LocalDisk::cleanup_tmp_on_startup(dir.path()).await.unwrap();
+        LocalDisk::cleanup_tmp_on_startup(dir.path(), Arc::new(AtomicU32::new(0)), Arc::new(Notify::new()))
+            .await
+            .unwrap();

        assert!(!tmp.join("leftover").exists());
        assert!(LocalDisk::meta_path(dir.path(), RUSTFS_META_TMP_DELETED_BUCKET).exists());
@@ -3213,6 +3285,54 @@ mod test {
        assert!(entries.next_entry().await.unwrap().is_none());
    }

+    #[tokio::test(start_paused = true)]
+    async fn cleanup_loop_interval_does_not_tick_immediately() {
+        let start_at = tokio::time::Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL;
+        let mut interval = interval_at(start_at, DELETED_OBJECTS_CLEANUP_INTERVAL);
+
+        assert!(tokio::time::timeout(Duration::from_secs(1), interval.tick()).await.is_err());
+
+        tokio::time::advance(DELETED_OBJECTS_CLEANUP_INTERVAL).await;
+        interval.tick().await;
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn startup_cleanup_barrier_waits_for_notification() {
+        let ready = Arc::new(AtomicU32::new(0));
+        let notify = Arc::new(Notify::new());
+
+        let wait = tokio::spawn({
+            let ready = ready.clone();
+            let notify = notify.clone();
+            async move { wait_for_startup_cleanup_signal(ready.as_ref(), notify.as_ref(), Duration::from_secs(2)).await }
+        });
+
+        tokio::task::yield_now().await;
+        assert!(!wait.is_finished());
+
+        ready.store(1, Ordering::Release);
+        notify.notify_waiters();
+
+        assert!(wait.await.unwrap());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn startup_cleanup_barrier_times_out() {
+        let ready = Arc::new(AtomicU32::new(0));
+        let notify = Arc::new(Notify::new());
+
+        let wait = tokio::spawn({
+            let ready = ready.clone();
+            let notify = notify.clone();
+            async move { wait_for_startup_cleanup_signal(ready.as_ref(), notify.as_ref(), Duration::from_secs(2)).await }
+        });
+
+        tokio::task::yield_now().await;
+        tokio::time::advance(Duration::from_secs(2)).await;
+
+        assert!(!wait.await.unwrap());
+    }
+
    #[tokio::test]
    async fn test_scan_dir_includes_nested_object_dirs() {
        use rustfs_filemeta::MetacacheReader;
@@ -226,7 +226,16 @@ impl Erasure {
                match rustfs_utils::read_full(&mut reader, &mut buf).await {
                    Ok(n) if n > 0 => {
                        total += n;
-                        let res = self.encode_data(&buf[..n])?;
+                        let erasure = self.clone();
+                        let encode_buf = std::mem::take(&mut buf);
+                        let (res, returned_buf) = tokio::task::spawn_blocking(move || {
+                            let res = erasure.encode_data(&encode_buf[..n]);
+                            (res, encode_buf)
+                        })
+                        .await
+                        .map_err(|err| std::io::Error::other(format!("EC encode task failed: {err}")))?;
+                        buf = returned_buf;
+                        let res = res?;
                        let queued_bytes = queued_block_bytes(&res);
                        rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
                        if let Err(err) = tx.send(res).await {
@@ -587,7 +587,21 @@ impl Erasure {
                Ok(n) if n > 0 => {
                    warn!("encode_stream_callback_async read n={}", n);
                    total += n;
-                    let res = self.encode_data(&buf[..n]);
+                    let erasure = self.clone();
+                    let encode_buf = std::mem::take(&mut buf);
+                    let (res, returned_buf) = match tokio::task::spawn_blocking(move || {
+                        let res = erasure.encode_data(&encode_buf[..n]);
+                        (res, encode_buf)
+                    })
+                    .await
+                    {
+                        Ok(result) => result,
+                        Err(err) => {
+                            on_block(Err(std::io::Error::other(format!("EC encode task failed: {err}")))).await?;
+                            break;
+                        }
+                    };
+                    buf = returned_buf;
                    on_block(res).await?
                }
                Ok(_) => {
@@ -160,6 +160,14 @@ pub enum StorageError {
    NotModified,
    #[error("Invalid range specified: {0}")]
    InvalidRangeSpec(String),
+    #[error("Namespace lock quorum unavailable for {mode} lock on {bucket}/{object}: required {required}, achieved {achieved}")]
+    NamespaceLockQuorumUnavailable {
+        mode: &'static str,
+        bucket: String,
+        object: String,
+        required: usize,
+        achieved: usize,
+    },

    // ── Generic ──────────────────────────────────────────────────────
    #[error("Unexpected error")]
@@ -204,6 +212,7 @@ impl StorageError {
                | StorageError::ErasureWriteQuorum
                | StorageError::InsufficientReadQuorum(_, _)
                | StorageError::InsufficientWriteQuorum(_, _)
+                | StorageError::NamespaceLockQuorumUnavailable { .. }
        )
    }
 }
@@ -433,6 +442,19 @@ impl Clone for StorageError {
            StorageError::NotModified => StorageError::NotModified,
            StorageError::InvalidPartNumber(a) => StorageError::InvalidPartNumber(*a),
            StorageError::InvalidRangeSpec(a) => StorageError::InvalidRangeSpec(a.clone()),
+            StorageError::NamespaceLockQuorumUnavailable {
+                mode,
+                bucket,
+                object,
+                required,
+                achieved,
+            } => StorageError::NamespaceLockQuorumUnavailable {
+                mode,
+                bucket: bucket.clone(),
+                object: object.clone(),
+                required: *required,
+                achieved: *achieved,
+            },
        }
    }
 }
@@ -505,6 +527,7 @@ impl StorageError {
            StorageError::InvalidRangeSpec(_) => 0x3D,
            StorageError::NotModified => 0x3E,
            StorageError::InvalidPartNumber(_) => 0x3F,
+            StorageError::NamespaceLockQuorumUnavailable { .. } => 0x42,
        }
    }

@@ -579,6 +602,13 @@ impl StorageError {
            0x3D => Some(StorageError::InvalidRangeSpec(Default::default())),
            0x3E => Some(StorageError::NotModified),
            0x3F => Some(StorageError::InvalidPartNumber(Default::default())),
+            0x42 => Some(StorageError::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: Default::default(),
+                object: Default::default(),
+                required: Default::default(),
+                achieved: Default::default(),
+            }),
            _ => None,
        }
    }
@@ -984,6 +1014,17 @@ mod tests {
        assert_eq!(StorageError::DecommissionAlreadyRunning.to_u32(), 0x30);
        assert_eq!(StorageError::RebalanceAlreadyRunning.to_u32(), 0x40);
        assert_eq!(StorageError::OperationCanceled.to_u32(), 0x41);
+        assert_eq!(
+            StorageError::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: "bucket".into(),
+                object: "object".into(),
+                required: 3,
+                achieved: 2,
+            }
+            .to_u32(),
+            0x42
+        );
    }

    #[test]
@@ -998,6 +1039,10 @@ mod tests {
        assert!(matches!(StorageError::from_u32(0x30), Some(StorageError::DecommissionAlreadyRunning)));
        assert!(matches!(StorageError::from_u32(0x40), Some(StorageError::RebalanceAlreadyRunning)));
        assert!(matches!(StorageError::from_u32(0x41), Some(StorageError::OperationCanceled)));
+        assert!(matches!(
+            StorageError::from_u32(0x42),
+            Some(StorageError::NamespaceLockQuorumUnavailable { .. })
+        ));

        // Test invalid code returns None
        assert!(StorageError::from_u32(0xFF).is_none());
@@ -175,6 +175,10 @@ impl RemoteDisk {
        });
    }

+    fn mark_suspect_or_offline(&self, reason: &'static str) -> bool {
+        self.health.mark_failure(&self.endpoint, reason)
+    }
+
    /// Enable health monitoring after disk creation.
    /// Used to defer health checks until after startup format loading completes,
    /// so that remote peers have time to come online.
@@ -202,7 +206,10 @@ impl RemoteDisk {
        let mut interval = time::interval(get_drive_active_check_interval());

        // Perform basic connectivity check
-        if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") {
+        let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
+        if initial_probe_ok {
+            health.record_operation_success(&endpoint, "connectivity_probe_success");
+        } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
            warn!("Remote disk health check failed for {}: marking as faulty", addr);

            // Start recovery monitoring
@@ -245,7 +252,9 @@ impl RemoteDisk {
                    }

                    // Perform basic connectivity check
-                    if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") {
+                    if Self::perform_connectivity_check(&addr).await.is_ok() {
+                        health.record_operation_success(&endpoint, "connectivity_probe_success");
+                    } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                        warn!("Remote disk health check failed for {}: marking as faulty", addr);

                        // Start recovery monitoring
@@ -286,7 +295,7 @@ impl RemoteDisk {
                            return;
                        }
                    } else {
-                        health.mark_offline(&endpoint, "connectivity_probe_failed");
+                        health.mark_failure(&endpoint, "connectivity_probe_failed");
                    }
                }
            }
@@ -440,7 +449,11 @@ impl RemoteDisk {
    }

    async fn mark_faulty_and_evict(&self, reason: &'static str) {
-        if self.health.mark_offline(&self.endpoint, reason) {
+        let previous_state = self.runtime_state();
+        let transitioned_to_offline = self.mark_suspect_or_offline(reason);
+        let state = self.runtime_state();
+
+        if state != previous_state {
            self.spawn_recovery_monitor_if_needed();
            counter!(
                "rustfs_drive_faulty_mark_total",
@@ -448,10 +461,17 @@ impl RemoteDisk {
                "reason" => reason.to_string()
            )
            .increment(1);
-            warn!(
-                "Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}",
-                self.endpoint, self.addr, reason
-            );
+            if transitioned_to_offline {
+                warn!(
+                    "Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}",
+                    self.endpoint, self.addr, reason
+                );
+            } else {
+                warn!(
+                    "Remote disk marked suspect after timeout: endpoint={}, addr={}, reason={}, state={:?}",
+                    self.endpoint, self.addr, reason, state
+                );
+            }
            counter!(
                "rustfs_drive_connection_evict_total",
                "endpoint" => self.endpoint.to_string(),
@@ -1951,20 +1971,39 @@ mod tests {
            disk_idx: 0,
        };

-        let disk_option = DiskOption {
-            cleanup: false,
-            health_check: true,
-        };
+        temp_env::async_with_vars(
+            [
+                (rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, Some("1")),
+                (rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, Some("1")),
+            ],
+            async {
+                let disk_option = DiskOption {
+                    cleanup: false,
+                    health_check: true,
+                };

-        let remote_disk = RemoteDisk::new(&endpoint, &disk_option, Arc::new(TcpHttpInternodeDataTransport))
-            .await
-            .unwrap();
-        remote_disk.enable_health_check();
+                let remote_disk = RemoteDisk::new(&endpoint, &disk_option, Arc::new(TcpHttpInternodeDataTransport))
+                    .await
+                    .unwrap();
+                remote_disk.enable_health_check();

-        // wait for health check connect timeout
-        tokio::time::sleep(Duration::from_secs(6)).await;
-
-        assert!(!remote_disk.is_online().await);
+                // Wait out the initial success-grace window so the active probe loop
+                // actually attempts a connectivity check. Under the new
+                // suspect-first semantics we only need to prove that the drive
+                // transitions away from a clean Online state at least once.
+                tokio::time::sleep(SKIP_IF_SUCCESS_BEFORE + Duration::from_secs(2)).await;
+                assert!(
+                    remote_disk.offline_duration_secs().is_some(),
+                    "missing listener should transition the drive through suspect/offline tracking"
+                );
+                assert_ne!(
+                    remote_disk.runtime_state(),
+                    RuntimeDriveHealthState::Online,
+                    "missing listener should not remain in a clean Online state after probing"
+                );
+            },
+        )
+        .await;
    }

    #[tokio::test]
@@ -2284,7 +2323,12 @@ mod tests {
            .expect_err("timeout should fail");

        assert!(err.to_string().contains("timeout"));
-        assert!(!remote_disk.is_online().await, "remote disk should be marked faulty after timeout");
+        assert!(remote_disk.is_online().await, "first timeout should keep the remote disk online");
+        assert_eq!(
+            remote_disk.runtime_state(),
+            RuntimeDriveHealthState::Suspect,
+            "first timeout should move the remote disk into suspect state"
+        );
    }

    #[tokio::test]
@@ -2450,7 +2494,15 @@ mod tests {
            },
            std::io::ErrorKind::TimedOut
        );
-        assert!(!remote_disk.is_online().await, "timeout-like errors should mark remote disk faulty");
+        assert!(
+            remote_disk.is_online().await,
+            "first timeout-like error should keep the remote disk online"
+        );
+        assert_eq!(
+            remote_disk.runtime_state(),
+            RuntimeDriveHealthState::Suspect,
+            "first timeout-like error should move the remote disk into suspect state"
+        );
        assert!(
            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
            "timeout-like errors should evict cached connection"
@@ -2503,7 +2555,15 @@ mod tests {
            },
            std::io::ErrorKind::ConnectionRefused
        );
-        assert!(!remote_disk.is_online().await, "network-like errors should mark remote disk faulty");
+        assert!(
+            remote_disk.is_online().await,
+            "first network-like error should keep the remote disk online"
+        );
+        assert_eq!(
+            remote_disk.runtime_state(),
+            RuntimeDriveHealthState::Suspect,
+            "first network-like error should move the remote disk into suspect state"
+        );
        assert!(
            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
            "network-like errors should evict cached connection"
@@ -14,18 +14,21 @@

 use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
 use async_trait::async_trait;
+use bytes::Bytes;
 use rustfs_lock::{
    LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
    types::{LockId, LockMetadata, LockPriority},
 };
 use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
-use rustfs_protos::{evict_failed_connection, proto_gen::node_service::node_service_client::NodeServiceClient};
+use rustfs_protos::{
+    evict_failed_connection, models::PingBodyBuilder, proto_gen::node_service::node_service_client::NodeServiceClient,
+};
 use std::time::Duration;
 use tokio::time::timeout;
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
 use tonic::transport::Channel;
-use tracing::{info, warn};
+use tracing::{debug, info, warn};

 /// Remote lock client implementation
 #[derive(Debug, Clone)]
@@ -42,6 +45,20 @@ impl RemoteClient {
        Self { addr: url.to_string() }
    }

+    fn build_ping_request() -> PingRequest {
+        let mut fbb = flatbuffers::FlatBufferBuilder::new();
+        let payload = fbb.create_vector(b"health-check");
+        let mut builder = PingBodyBuilder::new(&mut fbb);
+        builder.add_payload(payload);
+        let root = builder.finish();
+        fbb.finish(root, None);
+
+        PingRequest {
+            version: 1,
+            body: Bytes::copy_from_slice(fbb.finished_data()),
+        }
+    }
+
    /// Create a minimal LockRequest for unlock operations using only lock_id
    fn create_unlock_request(lock_id: &LockId) -> LockRequest {
        LockRequest {
@@ -64,16 +81,44 @@ impl RemoteClient {
            .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))
    }

-    async fn evict_connection(&self, op: &'static str, reason: &str) {
-        warn!(
-            addr = %self.addr,
-            op,
-            reason,
-            "Evicting cached remote lock connection after RPC failure"
-        );
+    fn is_scanner_leader_lock(resource_summary: &str) -> bool {
+        resource_summary == ".rustfs.sys/leader.lock@latest"
+    }
+
+    async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str) {
+        if Self::is_scanner_leader_lock(resource_summary) {
+            debug!(
+                addr = %self.addr,
+                op,
+                reason,
+                resource_summary,
+                "Evicting cached remote lock connection for scanner leader-lock RPC failure"
+            );
+        } else {
+            warn!(
+                addr = %self.addr,
+                op,
+                reason,
+                resource_summary,
+                "Evicting cached remote lock connection after RPC failure"
+            );
+        }
        evict_failed_connection(&self.addr).await;
    }

+    fn summarize_resources(requests: &[LockRequest]) -> String {
+        const LIMIT: usize = 3;
+        let mut resources = requests
+            .iter()
+            .take(LIMIT)
+            .map(|request| request.resource.to_string())
+            .collect::<Vec<_>>();
+        if requests.len() > LIMIT {
+            resources.push(format!("... (+{} more)", requests.len() - LIMIT));
+        }
+        resources.join(", ")
+    }
+
    fn rpc_timeout(timeout_duration: Duration) -> Duration {
        if timeout_duration.is_zero() {
            Duration::from_millis(1)
@@ -86,39 +131,74 @@ impl RemoteClient {
        &self,
        op: &'static str,
        timeout_duration: Duration,
+        resource_summary: &str,
        future: F,
    ) -> std::result::Result<T, LockError>
    where
        F: std::future::Future<Output = std::result::Result<T, tonic::Status>>,
    {
-        let timeout_duration = Self::rpc_timeout(timeout_duration);
-        match timeout(timeout_duration, future).await {
+        let lock_timeout = Self::rpc_timeout(timeout_duration);
+        match timeout(lock_timeout, future).await {
            Ok(Ok(response)) => Ok(response),
            Ok(Err(err)) => {
                let reason = err.to_string();
-                self.evict_connection(op, &reason).await;
+                if Self::is_scanner_leader_lock(resource_summary) {
+                    debug!(
+                        addr = %self.addr,
+                        op,
+                        timeout_ms = lock_timeout.as_millis(),
+                        resource_summary,
+                        tonic_code = ?err.code(),
+                        tonic_message = err.message(),
+                        "Remote lock RPC returned tonic error for scanner leader lock"
+                    );
+                } else {
+                    warn!(
+                        addr = %self.addr,
+                        op,
+                        timeout_ms = lock_timeout.as_millis(),
+                        resource_summary,
+                        tonic_code = ?err.code(),
+                        tonic_message = err.message(),
+                        "Remote lock RPC returned tonic error"
+                    );
+                }
+                self.evict_connection(op, &reason, resource_summary).await;
                Err(LockError::internal(format!("{op} RPC failed: {reason}")))
            }
            Err(_) => {
-                let reason = format!("RPC timed out after {:?}", timeout_duration);
-                self.evict_connection(op, &reason).await;
-                Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), timeout_duration))
+                let reason = format!("RPC timed out after {:?}", lock_timeout);
+                if Self::is_scanner_leader_lock(resource_summary) {
+                    debug!(
+                        addr = %self.addr,
+                        op,
+                        timeout_ms = lock_timeout.as_millis(),
+                        resource_summary,
+                        "Remote lock RPC timed out for scanner leader lock"
+                    );
+                } else {
+                    warn!(
+                        addr = %self.addr,
+                        op,
+                        timeout_ms = lock_timeout.as_millis(),
+                        resource_summary,
+                        "Remote lock RPC timed out"
+                    );
+                }
+                self.evict_connection(op, &reason, resource_summary).await;
+                Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), lock_timeout))
            }
        }
    }

-    fn timeout_failure_response(request: &LockRequest) -> LockResponse {
-        LockResponse::failure("Lock acquisition timeout", request.acquire_timeout)
+    fn rpc_timeout_failure_response(request: &LockRequest, err: &LockError) -> LockResponse {
+        LockResponse::failure(format!("Remote lock RPC timed out: {err}"), request.acquire_timeout)
    }

    fn rpc_failure_response(_request: &LockRequest, err: &LockError) -> LockResponse {
        LockResponse::failure(format!("Remote lock RPC failed: {err}"), Duration::ZERO)
    }

-    fn timeout_failure_batch(requests: &[LockRequest]) -> Vec<LockResponse> {
-        requests.iter().map(Self::timeout_failure_response).collect()
-    }
-
    fn rpc_failure_batch(requests: &[LockRequest], err: &LockError) -> Vec<LockResponse> {
        requests
            .iter()
@@ -126,6 +206,13 @@ impl RemoteClient {
            .collect()
    }

+    fn rpc_timeout_failure_batch(requests: &[LockRequest], err: &LockError) -> Vec<LockResponse> {
+        requests
+            .iter()
+            .map(|request| Self::rpc_timeout_failure_response(request, err))
+            .collect()
+    }
+
    fn batch_rpc_timeout(requests: &[LockRequest]) -> Duration {
        requests
            .iter()
@@ -179,14 +266,18 @@ impl LockClient for RemoteClient {
    async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
        info!("remote acquire_exclusive for {}", request.resource);
        let mut client = self.get_client().await?;
+        let resource_summary = request.resource.to_string();
        let req = Request::new(GenerallyLockRequest {
            args: serde_json::to_string(&request)
                .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
        });

-        let resp = match self.execute_rpc("lock", request.acquire_timeout, client.lock(req)).await {
+        let resp = match self
+            .execute_rpc("lock", request.acquire_timeout, &resource_summary, client.lock(req))
+            .await
+        {
            Ok(resp) => resp.into_inner(),
-            Err(LockError::Timeout { .. }) => return Ok(Self::timeout_failure_response(request)),
+            Err(err @ LockError::Timeout { .. }) => return Ok(Self::rpc_timeout_failure_response(request, &err)),
            Err(err) => return Ok(Self::rpc_failure_response(request, &err)),
        };

@@ -212,6 +303,7 @@ impl LockClient for RemoteClient {
        }

        let mut client = self.get_client().await?;
+        let resource_summary = Self::summarize_resources(requests);
        let req = Request::new(BatchGenerallyLockRequest {
            args: requests
                .iter()
@@ -222,11 +314,11 @@ impl LockClient for RemoteClient {
        });

        let resp = match self
-            .execute_rpc("lock_batch", Self::batch_rpc_timeout(requests), client.lock_batch(req))
+            .execute_rpc("lock_batch", Self::batch_rpc_timeout(requests), &resource_summary, client.lock_batch(req))
            .await
        {
            Ok(resp) => resp.into_inner(),
-            Err(LockError::Timeout { .. }) => return Ok(Self::timeout_failure_batch(requests)),
+            Err(err @ LockError::Timeout { .. }) => return Ok(Self::rpc_timeout_failure_batch(requests, &err)),
            Err(err) => return Ok(Self::rpc_failure_batch(requests, &err)),
        };

@@ -435,10 +527,7 @@ impl LockClient for RemoteClient {
            }
        };

-        let ping_req = Request::new(PingRequest {
-            version: 1,
-            body: bytes::Bytes::new(),
-        });
+        let ping_req = Request::new(Self::build_ping_request());

        match client.ping(ping_req).await {
            Ok(_) => {
@@ -511,7 +600,14 @@ mod tests {
            "remote lock RPC should honor request timeout"
        );
        assert!(!response.success, "timed out lock acquisition should fail");
-        assert_eq!(response.error.as_deref(), Some("Lock acquisition timeout"));
+        assert!(
+            response
+                .error
+                .as_deref()
+                .is_some_and(|error| error.contains("Remote lock RPC timed out")),
+            "expected remote RPC timeout marker, got {:?}",
+            response.error
+        );
        assert!(
            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
            "timeout should evict cached connection"
@@ -539,7 +635,14 @@ mod tests {
        );
        assert_eq!(responses.len(), 1);
        assert!(!responses[0].success, "timed out batch lock acquisition should fail");
-        assert_eq!(responses[0].error.as_deref(), Some("Lock acquisition timeout"));
+        assert!(
+            responses[0]
+                .error
+                .as_deref()
+                .is_some_and(|error| error.contains("Remote lock RPC timed out")),
+            "expected remote RPC timeout marker, got {:?}",
+            responses[0].error
+        );
        assert!(
            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
            "batch timeout should evict cached connection"
@@ -622,12 +622,7 @@ impl ObjectIO for SetDisks {
                .await?
                .get_read_lock(get_lock_acquire_timeout())
                .await
-                .map_err(|e| {
-                    Error::other(format!(
-                        "Failed to acquire read lock: {}",
-                        self.format_lock_error_from_error(bucket, object, "read", &e)
-                    ))
-                })?;
+                .map_err(|e| self.map_namespace_lock_error(bucket, object, "read", e))?;

            // Record lock acquisition for deadlock detection
            let _lock_id = record_lock_acquire(bucket, object, "read");
@@ -756,12 +751,12 @@ impl ObjectIO for SetDisks {
        if let Some(http_preconditions) = opts.http_preconditions.clone() {
            if !opts.no_lock {
                let ns_lock = self.new_ns_lock(bucket, object).await?;
-                object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
-                    StorageError::other(format!(
-                        "Failed to acquire write lock: {}",
-                        self.format_lock_error_from_error(bucket, object, "write", &e)
-                    ))
-                })?);
+                object_lock_guard = Some(
+                    ns_lock
+                        .get_write_lock(get_lock_acquire_timeout())
+                        .await
+                        .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
+                );
            }

            if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
@@ -975,12 +970,12 @@ impl ObjectIO for SetDisks {

            if !opts.no_lock && object_lock_guard.is_none() {
                let ns_lock = self.new_ns_lock(bucket, object).await?;
-                object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
-                    StorageError::other(format!(
-                        "Failed to acquire write lock: {}",
-                        self.format_lock_error_from_error(bucket, object, "write", &e)
-                    ))
-                })?);
+                object_lock_guard = Some(
+                    ns_lock
+                        .get_write_lock(get_lock_acquire_timeout())
+                        .await
+                        .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
+                );
            }

            let (online_disks, _, op_old_dir) = Self::rename_data(
@@ -1398,12 +1393,7 @@ impl ObjectOperations for SetDisks {
            .await?
            .get_write_lock(get_lock_acquire_timeout())
            .await
-            .map_err(|e| {
-                Error::other(format!(
-                    "Failed to acquire write lock: {}",
-                    self.format_lock_error_from_error(src_bucket, src_object, "write", &e)
-                ))
-            })?;
+            .map_err(|e| self.map_namespace_lock_error(src_bucket, src_object, "write", e))?;

        let disks = self.get_disks_internal().await;

@@ -1785,12 +1775,7 @@ impl ObjectOperations for SetDisks {
                    .await?
                    .get_write_lock(get_lock_acquire_timeout())
                    .await
-                    .map_err(|e| {
-                        Error::other(format!(
-                            "Failed to acquire write lock: {}",
-                            self.format_lock_error_from_error(bucket, object, "write", &e)
-                        ))
-                    })?,
+                    .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
            )
        } else {
            None
@@ -1931,12 +1916,7 @@ impl ObjectOperations for SetDisks {
                    .await?
                    .get_read_lock(get_lock_acquire_timeout())
                    .await
-                    .map_err(|e| {
-                        Error::other(format!(
-                            "Failed to acquire read lock: {}",
-                            self.format_lock_error_from_error(bucket, object, "read", &e)
-                        ))
-                    })?,
+                    .map_err(|e| self.map_namespace_lock_error(bucket, object, "read", e))?,
            )
        } else {
            None
@@ -1990,12 +1970,7 @@ impl ObjectOperations for SetDisks {
                    .await?
                    .get_write_lock(get_lock_acquire_timeout())
                    .await
-                    .map_err(|e| {
-                        Error::other(format!(
-                            "Failed to acquire write lock: {}",
-                            self.format_lock_error_from_error(bucket, object, "write", &e)
-                        ))
-                    })?,
+                    .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
            )
        } else {
            None
@@ -2527,12 +2502,7 @@ impl SetDisks {
                    .await?
                    .get_write_lock(get_lock_acquire_timeout())
                    .await
-                    .map_err(|e| {
-                        Error::other(format!(
-                            "Failed to acquire write lock: {}",
-                            self.format_lock_error_from_error(bucket, object, "write", &e)
-                        ))
-                    })?,
+                    .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
            )
        } else {
            None
@@ -3067,12 +3037,12 @@ impl MultipartOperations for SetDisks {
        if let Some(http_preconditions) = opts.http_preconditions.clone() {
            let object_lock_guard = if !opts.no_lock {
                let ns_lock = self.new_ns_lock(bucket, object).await?;
-                Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
-                    StorageError::other(format!(
-                        "Failed to acquire write lock: {}",
-                        self.format_lock_error_from_error(bucket, object, "write", &e)
-                    ))
-                })?)
+                Some(
+                    ns_lock
+                        .get_write_lock(get_lock_acquire_timeout())
+                        .await
+                        .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
+                )
            } else {
                None
            };
@@ -3249,12 +3219,12 @@ impl MultipartOperations for SetDisks {
        if let Some(http_preconditions) = opts.http_preconditions.clone() {
            if !opts.no_lock {
                let ns_lock = self.new_ns_lock(bucket, object).await?;
-                object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
-                    StorageError::other(format!(
-                        "Failed to acquire write lock: {}",
-                        self.format_lock_error_from_error(bucket, object, "write", &e)
-                    ))
-                })?);
+                object_lock_guard = Some(
+                    ns_lock
+                        .get_write_lock(get_lock_acquire_timeout())
+                        .await
+                        .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
+                );
            }

            if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
@@ -3599,12 +3569,12 @@ impl MultipartOperations for SetDisks {

        if !opts.no_lock && object_lock_guard.is_none() {
            let ns_lock = self.new_ns_lock(bucket, object).await?;
-            object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
-                StorageError::other(format!(
-                    "Failed to acquire write lock: {}",
-                    self.format_lock_error_from_error(bucket, object, "write", &e)
-                ))
-            })?);
+            object_lock_guard = Some(
+                ns_lock
+                    .get_write_lock(get_lock_acquire_timeout())
+                    .await
+                    .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
+            );
        }

        self.cleanup_multipart_path(&parts).await;
@@ -3685,12 +3655,12 @@ impl HealOperations for SetDisks {
    ) -> Result<(HealResultItem, Option<Error>)> {
        let _write_lock_guard = if !opts.no_lock {
            let ns_lock = self.new_ns_lock(bucket, object).await?;
-            Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
-                StorageError::other(format!(
-                    "Failed to acquire write lock: {}",
-                    self.format_lock_error_from_error(bucket, object, "write", &e)
-                ))
-            })?)
+            Some(
+                ns_lock
+                    .get_write_lock(get_lock_acquire_timeout())
+                    .await
+                    .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
+            )
        } else {
            None
        };
@@ -39,12 +39,12 @@ impl SetDisks {

        let write_lock_guard = if !opts.no_lock {
            let ns_lock = self.new_ns_lock(bucket, object).await?;
-            Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
-                StorageError::other(format!(
-                    "Failed to acquire write lock: {}",
-                    self.format_lock_error_from_error(bucket, object, "write", &e)
-                ))
-            })?)
+            Some(
+                ns_lock
+                    .get_write_lock(get_lock_acquire_timeout())
+                    .await
+                    .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
+            )
        } else {
            None
        };
@@ -713,13 +713,7 @@ impl SetDisks {
            .await?
            .get_write_lock(get_lock_acquire_timeout())
            .await
-            .map_err(|e| {
-                let message = format!(
-                    "Failed to acquire write lock: {}",
-                    self.format_lock_error_from_error(bucket, object, "write", &e)
-                );
-                DiskError::other(message)
-            })?;
+            .map_err(|e| DiskError::other(self.map_namespace_lock_error(bucket, object, "write", e).to_string()))?;

        self.heal_object_dir_locked(bucket, object, dry_run, remove).await
    }
@@ -50,6 +50,30 @@ impl SetDisks {
        }
    }

+    pub(super) fn map_namespace_lock_error(
+        &self,
+        bucket: &str,
+        object: &str,
+        mode: &'static str,
+        err: rustfs_lock::error::LockError,
+    ) -> StorageError {
+        match err {
+            rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
+                StorageError::NamespaceLockQuorumUnavailable {
+                    mode,
+                    bucket: bucket.to_string(),
+                    object: object.to_string(),
+                    required,
+                    achieved,
+                }
+            }
+            other => StorageError::other(format!(
+                "Failed to acquire {mode} lock: {}",
+                self.format_lock_error_from_error(bucket, object, mode, &other)
+            )),
+        }
+    }
+
    pub(super) async fn get_disks_internal(&self) -> Vec<Option<DiskStore>> {
        let rl = self.disks.read().await;

@@ -262,6 +262,11 @@ impl SetDisks {
        let dst_bucket = Arc::new(dst_bucket.to_string());
        let dst_object = Arc::new(dst_object.to_string());

+        // Match MinIO's multipart overwrite semantics: clear any stale destination
+        // part payload and metadata before the new per-disk rename fan-out begins.
+        self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])
+            .await;
+
        let mut errs = Vec::with_capacity(disks.len());

        let futures = disks.iter().map(|disk| {
@@ -384,14 +384,20 @@ impl ObjectInfo {
    }

    pub fn is_encrypted(&self) -> bool {
+        // Corresponding to the logic in rustfs/src/sse.rs/encryption_material_to_metadata function
        use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};

        self.user_defined
            .keys()
-            .any(|key| rustfs_utils::http::is_encryption_metadata_key(key))
-            || self.user_defined.contains_key(SSEC_ALGORITHM_HEADER)
-            || self.user_defined.contains_key(SSEC_KEY_HEADER)
-            || self.user_defined.contains_key(SSEC_KEY_MD5_HEADER)
+            .any(|key| key.to_lowercase().starts_with("x-minio-encryption-")) // Covers MinIO metadata
+            || self.user_defined.contains_key("x-rustfs-encryption-key") // SSE-S3/SSE-KMS
+            || self.user_defined.contains_key("x-rustfs-encryption-algorithm") // SSE-S3/SSE-KMS
+            || self.user_defined.contains_key("x-rustfs-encryption-iv") // SSE-S3/SSE-KMS
+            || self.user_defined.contains_key("x-amz-server-side-encryption-aws-kms-key-id") // SSE-KMS
+            || self.user_defined.contains_key(SSEC_ALGORITHM_HEADER) // SSE-C
+            || self.user_defined.contains_key(SSEC_KEY_HEADER) // SSE-C
+            || self.user_defined.contains_key(SSEC_KEY_MD5_HEADER) // SSE-C
+            || self.user_defined.contains_key("x-amz-server-side-encryption") // SSE-S3/SSE-KMS/SSE-C
    }

    pub fn encryption_original_size(&self) -> std::io::Result<Option<i64>> {
@@ -1193,4 +1199,58 @@ mod tests {

        assert!(info.get_actual_size().is_err());
    }
+
+    #[test]
+    fn is_encrypted_correct_for_old_version_fileinfo() {
+        let mut user_defined: HashMap<String, String> = HashMap::new();
+
+        let metadata = vec![
+            ("content-type", "text/plain"),
+            ("etag", "e4336b5de4e2180a53fe2e17d03abe4f-4"),
+            ("x-minio-internal-actual-size", "67108864"),
+            ("x-rustfs-encryption-original-size", "67108864"),
+            ("x-rustfs-internal-actual-size", "67108864"),
+        ];
+
+        metadata.into_iter().for_each(|(key, value)| {
+            user_defined.insert(key.to_string(), value.to_string());
+        });
+
+        let info = ObjectInfo {
+            user_defined,
+            ..Default::default()
+        };
+
+        assert!(!info.is_encrypted());
+    }
+
+    #[test]
+    fn is_encrypted_returns_true_when_encryption_metadata_present() {
+        let mut user_defined: HashMap<String, String> = HashMap::new();
+
+        let metadata = vec![
+            ("content-type", "text/plain"),
+            ("etag", "f1c9645dbc14efddc7d8a322685f26eb"),
+            ("x-amz-server-side-encryption", "AES256"),
+            ("x-rustfs-encryption-algorithm", "AES256"),
+            ("x-rustfs-encryption-iv", "Fb9moBlEBRE0D14F"),
+            (
+                "x-rustfs-encryption-key",
+                "QUFBQUFBQUFBQUFBQUFBQTpZQk5sNnNJdmJHWWl3QmxZbCtsMTJlVlZCeXVoVml4UlV4b3JPbTNoRk5odUlYVnBPdlpXNWVyT0FTcklXMWJr",
+            ),
+            ("x-rustfs-encryption-key-id", "default"),
+            ("x-rustfs-encryption-original-size", "10485760"),
+        ];
+
+        metadata.into_iter().for_each(|(key, value)| {
+            user_defined.insert(key.to_string(), value.to_string());
+        });
+
+        let info = ObjectInfo {
+            user_defined,
+            ..Default::default()
+        };
+
+        assert!(info.is_encrypted());
+    }
 }
@@ -16,10 +16,10 @@ use std::{
    collections::{HashMap, HashSet},
    ops::{Deref, DerefMut},
    ptr,
-    sync::Arc,
+    sync::{Arc, Mutex},
 };

-use arc_swap::{ArcSwap, AsRaw, Guard};
+use arc_swap::{ArcSwap, Guard};
 use rustfs_policy::{
    auth::UserIdentity,
    policy::{Args, PolicyDoc},
@@ -29,80 +29,302 @@ use tracing::warn;

 use crate::store::{GroupInfo, MappedPolicy};

+/// Immutable IAM cache snapshot published atomically by [`Cache`].
+///
+/// Readers should load one `CacheState` and read all related maps from it. Writers
+/// must go through `Cache`/`LockedCache` so multi-map updates publish as one state.
+#[derive(Clone)]
+pub(crate) struct CacheState {
+    pub(crate) policy_docs: Arc<CacheEntity<PolicyDoc>>,
+    pub(crate) users: Arc<CacheEntity<UserIdentity>>,
+    pub(crate) user_policies: Arc<CacheEntity<MappedPolicy>>,
+    pub(crate) sts_accounts: Arc<CacheEntity<UserIdentity>>,
+    pub(crate) sts_policies: Arc<CacheEntity<MappedPolicy>>,
+    pub(crate) groups: Arc<CacheEntity<GroupInfo>>,
+    pub(crate) user_group_memberships: Arc<CacheEntity<HashSet<String>>>,
+    pub(crate) group_policies: Arc<CacheEntity<MappedPolicy>>,
+}
+
+impl Default for CacheState {
+    fn default() -> Self {
+        Self {
+            policy_docs: Arc::new(CacheEntity::default()),
+            users: Arc::new(CacheEntity::default()),
+            user_policies: Arc::new(CacheEntity::default()),
+            sts_accounts: Arc::new(CacheEntity::default()),
+            sts_policies: Arc::new(CacheEntity::default()),
+            groups: Arc::new(CacheEntity::default()),
+            user_group_memberships: Arc::new(CacheEntity::default()),
+            group_policies: Arc::new(CacheEntity::default()),
+        }
+    }
+}
+
 pub struct Cache {
-    pub policy_docs: ArcSwap<CacheEntity<PolicyDoc>>,
-    pub users: ArcSwap<CacheEntity<UserIdentity>>,
-    pub user_policies: ArcSwap<CacheEntity<MappedPolicy>>,
-    pub sts_accounts: ArcSwap<CacheEntity<UserIdentity>>,
-    pub sts_policies: ArcSwap<CacheEntity<MappedPolicy>>,
-    pub groups: ArcSwap<CacheEntity<GroupInfo>>,
-    pub user_group_memberships: ArcSwap<CacheEntity<HashSet<String>>>,
-    pub group_policies: ArcSwap<CacheEntity<MappedPolicy>>,
+    state: ArcSwap<CacheState>,
+    write_lock: Mutex<()>,
 }

 impl Default for Cache {
    fn default() -> Self {
        Self {
-            policy_docs: ArcSwap::new(Arc::new(CacheEntity::default())),
-            users: ArcSwap::new(Arc::new(CacheEntity::default())),
-            user_policies: ArcSwap::new(Arc::new(CacheEntity::default())),
-            sts_accounts: ArcSwap::new(Arc::new(CacheEntity::default())),
-            sts_policies: ArcSwap::new(Arc::new(CacheEntity::default())),
-            groups: ArcSwap::new(Arc::new(CacheEntity::default())),
-            user_group_memberships: ArcSwap::new(Arc::new(CacheEntity::default())),
-            group_policies: ArcSwap::new(Arc::new(CacheEntity::default())),
+            state: ArcSwap::new(Arc::new(CacheState::default())),
+            write_lock: Mutex::new(()),
        }
    }
 }

+pub(crate) type CacheSnapshot = Guard<Arc<CacheState>>;
+
 impl Cache {
-    pub fn ptr_eq<Base, A, B>(a: A, b: B) -> bool
-    where
-        A: AsRaw<Base>,
-        B: AsRaw<Base>,
-    {
-        let a = a.as_raw();
-        let b = b.as_raw();
-        ptr::eq(a, b)
+    pub(crate) fn snapshot(&self) -> CacheSnapshot {
+        self.state.load()
    }

-    fn exec<T: Clone>(target: &ArcSwap<CacheEntity<T>>, t: OffsetDateTime, mut op: impl FnMut(&mut CacheEntity<T>)) {
-        let mut cur = target.load();
-        loop {
-            // If the current update time is later than the execution time,
-            // the background task is loaded and the current operation does not need to be performed.
-            if cur.load_time >= t {
-                return;
-            }
-
-            let mut new = CacheEntity::clone(&cur);
-            op(&mut new);
-
-            // Replace content with CAS atoms
-            let prev = target.compare_and_swap(&*cur, Arc::new(new));
-            let swapped = Self::ptr_eq(&*cur, &*prev);
-            if swapped {
-                return;
-            } else {
-                cur = prev;
-            }
+    pub(crate) fn with_write_lock<R>(&self, f: impl FnOnce(&mut LockedCache) -> R) -> R {
+        let _guard = self.write_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+        let current = self.state.load_full();
+        let mut locked = LockedCache {
+            state: CacheState::clone(&current),
+            current_ptr: Arc::as_ptr(&current),
+            dirty: false,
+        };
+        let ret = f(&mut locked);
+        if locked.dirty {
+            self.state.store(Arc::new(locked.state));
        }
+        ret
    }

-    pub fn add_or_update<T: Clone>(target: &ArcSwap<CacheEntity<T>>, key: &str, value: &T, t: OffsetDateTime) {
-        Self::exec(target, t, |map: &mut CacheEntity<T>| {
+    pub fn add_or_update_policy_doc(&self, key: &str, value: &PolicyDoc, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.add_or_update_policy_doc(key, value, t));
+    }
+
+    pub fn add_or_update_user(&self, key: &str, value: &UserIdentity, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.add_or_update_user(key, value, t));
+    }
+
+    pub fn add_or_update_user_policy(&self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.add_or_update_user_policy(key, value, t));
+    }
+
+    pub fn add_or_update_sts_account(&self, key: &str, value: &UserIdentity, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.add_or_update_sts_account(key, value, t));
+    }
+
+    pub fn add_or_update_sts_policy(&self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.add_or_update_sts_policy(key, value, t));
+    }
+
+    pub fn add_or_update_group(&self, key: &str, value: &GroupInfo, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.add_or_update_group(key, value, t));
+    }
+
+    pub fn add_or_update_user_group_membership(&self, key: &str, value: &HashSet<String>, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.add_or_update_user_group_membership(key, value, t));
+    }
+
+    pub fn add_or_update_group_policy(&self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.add_or_update_group_policy(key, value, t));
+    }
+
+    pub fn delete_policy_doc(&self, key: &str, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.delete_policy_doc(key, t));
+    }
+
+    pub fn delete_user(&self, key: &str, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.delete_user(key, t));
+    }
+
+    pub fn delete_user_policy(&self, key: &str, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.delete_user_policy(key, t));
+    }
+
+    pub fn delete_sts_account(&self, key: &str, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.delete_sts_account(key, t));
+    }
+
+    pub fn delete_sts_policy(&self, key: &str, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.delete_sts_policy(key, t));
+    }
+
+    pub fn delete_group(&self, key: &str, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.delete_group(key, t));
+    }
+
+    pub fn delete_group_policy(&self, key: &str, t: OffsetDateTime) {
+        self.with_write_lock(|cache| cache.delete_group_policy(key, t));
+    }
+}
+
+pub(crate) struct LockedCache {
+    state: CacheState,
+    current_ptr: *const CacheState,
+    dirty: bool,
+}
+
+impl LockedCache {
+    pub(crate) fn state(&self) -> &CacheState {
+        &self.state
+    }
+
+    pub(crate) fn matches_snapshot(&self, snapshot: &CacheSnapshot) -> bool {
+        ptr::eq(self.current_ptr, Arc::as_ptr(snapshot))
+    }
+
+    fn exec<T: Clone>(target: &mut Arc<CacheEntity<T>>, t: OffsetDateTime, mut op: impl FnMut(&mut CacheEntity<T>)) -> bool {
+        if target.load_time >= t {
+            return false;
+        }
+
+        let mut new = CacheEntity::clone(target);
+        op(&mut new);
+        *target = Arc::new(new);
+        true
+    }
+
+    fn replaced<T>(value: CacheEntity<T>) -> Arc<CacheEntity<T>> {
+        Arc::new(value.update_load_time())
+    }
+
+    pub(crate) fn replace_policy_docs(&mut self, value: CacheEntity<PolicyDoc>) {
+        self.state.policy_docs = Self::replaced(value);
+        self.dirty = true;
+    }
+
+    pub(crate) fn replace_users(&mut self, value: CacheEntity<UserIdentity>) {
+        self.state.users = Self::replaced(value);
+        self.dirty = true;
+    }
+
+    pub(crate) fn replace_user_policies(&mut self, value: CacheEntity<MappedPolicy>) {
+        self.state.user_policies = Self::replaced(value);
+        self.dirty = true;
+    }
+
+    pub(crate) fn replace_sts_accounts(&mut self, value: CacheEntity<UserIdentity>) {
+        self.state.sts_accounts = Self::replaced(value);
+        self.dirty = true;
+    }
+
+    pub(crate) fn replace_sts_policies(&mut self, value: CacheEntity<MappedPolicy>) {
+        self.state.sts_policies = Self::replaced(value);
+        self.dirty = true;
+    }
+
+    pub(crate) fn replace_groups(&mut self, value: CacheEntity<GroupInfo>) {
+        self.state.groups = Self::replaced(value);
+        self.dirty = true;
+    }
+
+    pub(crate) fn replace_group_policies(&mut self, value: CacheEntity<MappedPolicy>) {
+        self.state.group_policies = Self::replaced(value);
+        self.dirty = true;
+    }
+
+    pub(crate) fn replace_user_group_memberships(&mut self, value: CacheEntity<HashSet<String>>) {
+        self.state.user_group_memberships = Self::replaced(value);
+        self.dirty = true;
+    }
+
+    pub(crate) fn add_or_update_policy_doc(&mut self, key: &str, value: &PolicyDoc, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.policy_docs, t, |map| {
            map.insert(key.to_string(), value.clone());
-        })
+        });
    }

-    pub fn delete<T: Clone>(target: &ArcSwap<CacheEntity<T>>, key: &str, t: OffsetDateTime) {
-        Self::exec(target, t, |map: &mut CacheEntity<T>| {
+    pub(crate) fn add_or_update_user(&mut self, key: &str, value: &UserIdentity, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.users, t, |map| {
+            map.insert(key.to_string(), value.clone());
+        });
+    }
+
+    pub(crate) fn add_or_update_user_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.user_policies, t, |map| {
+            map.insert(key.to_string(), value.clone());
+        });
+    }
+
+    pub(crate) fn add_or_update_sts_account(&mut self, key: &str, value: &UserIdentity, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.sts_accounts, t, |map| {
+            map.insert(key.to_string(), value.clone());
+        });
+    }
+
+    pub(crate) fn add_or_update_sts_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.sts_policies, t, |map| {
+            map.insert(key.to_string(), value.clone());
+        });
+    }
+
+    pub(crate) fn add_or_update_group(&mut self, key: &str, value: &GroupInfo, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.groups, t, |map| {
+            map.insert(key.to_string(), value.clone());
+        });
+    }
+
+    pub(crate) fn add_or_update_user_group_membership(&mut self, key: &str, value: &HashSet<String>, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.user_group_memberships, t, |map| {
+            map.insert(key.to_string(), value.clone());
+        });
+    }
+
+    pub(crate) fn add_or_update_group_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.group_policies, t, |map| {
+            map.insert(key.to_string(), value.clone());
+        });
+    }
+
+    pub(crate) fn delete_policy_doc(&mut self, key: &str, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.policy_docs, t, |map| {
            map.remove(key);
-        })
+        });
    }

-    pub fn build_user_group_memberships(&self) {
-        let groups = self.groups.load();
+    pub(crate) fn delete_user(&mut self, key: &str, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.users, t, |map| {
+            map.remove(key);
+        });
+    }
+
+    pub(crate) fn delete_user_policy(&mut self, key: &str, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.user_policies, t, |map| {
+            map.remove(key);
+        });
+    }
+
+    pub(crate) fn delete_user_group_membership(&mut self, key: &str, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.user_group_memberships, t, |map| {
+            map.remove(key);
+        });
+    }
+
+    pub(crate) fn delete_sts_account(&mut self, key: &str, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.sts_accounts, t, |map| {
+            map.remove(key);
+        });
+    }
+
+    pub(crate) fn delete_sts_policy(&mut self, key: &str, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.sts_policies, t, |map| {
+            map.remove(key);
+        });
+    }
+
+    pub(crate) fn delete_group(&mut self, key: &str, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.groups, t, |map| {
+            map.remove(key);
+        });
+    }
+
+    pub(crate) fn delete_group_policy(&mut self, key: &str, t: OffsetDateTime) {
+        self.dirty |= Self::exec(&mut self.state.group_policies, t, |map| {
+            map.remove(key);
+        });
+    }
+
+    pub(crate) fn build_user_group_memberships(&mut self) {
+        let groups = Arc::clone(&self.state.groups);
        let mut user_group_memberships = HashMap::new();
        for (group_name, group) in groups.iter() {
            for user_name in &group.members {
@@ -112,49 +334,19 @@ impl Cache {
                    .insert(group_name.clone());
            }
        }
-        self.user_group_memberships
-            .store(Arc::new(CacheEntity::new(user_group_memberships)));
+        self.replace_user_group_memberships(CacheEntity::new(user_group_memberships));
    }
 }

 impl CacheInner {
    #[inline]
    pub fn get_user(&self, user_name: &str) -> Option<&UserIdentity> {
-        self.users.get(user_name).or_else(|| self.sts_accounts.get(user_name))
+        self.snapshot
+            .users
+            .get(user_name)
+            .or_else(|| self.snapshot.sts_accounts.get(user_name))
    }

-    // fn get_policy(&self, _name: &str, _groups: &[String]) -> crate::Result<Vec<Policy>> {
-    //     todo!()
-    // }
-
-    // /// Return Ok(Some(parent_name)) when the user is temporary.
-    // /// Return Ok(None) for non-temporary users.
-    // fn is_temp_user(&self, user_name: &str) -> crate::Result<Option<&str>> {
-    //     let user = self
-    //         .get_user(user_name)
-    //         .ok_or_else(|| Error::NoSuchUser(user_name.to_owned()))?;
-
-    //     if user.credentials.is_temp() {
-    //         Ok(Some(&user.credentials.parent_user))
-    //     } else {
-    //         Ok(None)
-    //     }
-    // }
-
-    // /// Return Ok(Some(parent_name)) when the user is a temporary identity.
-    // /// Return Ok(None) when the user is not temporary.
-    // fn is_service_account(&self, user_name: &str) -> crate::Result<Option<&str>> {
-    //     let user = self
-    //         .get_user(user_name)
-    //         .ok_or_else(|| Error::NoSuchUser(user_name.to_owned()))?;
-
-    //     if user.credentials.is_service_account() {
-    //         Ok(Some(&user.credentials.parent_user))
-    //     } else {
-    //         Ok(None)
-    //     }
-    // }
-
    // todo
    pub fn is_allowed_sts(&self, _args: &Args, _parent: &str) -> bool {
        warn!("policy cache STS check path is not implemented");
@@ -223,30 +415,14 @@ impl<T> CacheEntity<T> {
    }
 }

-pub type G<T> = Guard<Arc<CacheEntity<T>>>;
-
 pub struct CacheInner {
-    pub policy_docs: G<PolicyDoc>,
-    pub users: G<UserIdentity>,
-    pub user_policies: G<MappedPolicy>,
-    pub sts_accounts: G<UserIdentity>,
-    pub sts_policies: G<MappedPolicy>,
-    pub groups: G<GroupInfo>,
-    pub user_group_memberships: G<HashSet<String>>,
-    pub group_policies: G<MappedPolicy>,
+    snapshot: CacheSnapshot,
 }

 impl From<&Cache> for CacheInner {
    fn from(value: &Cache) -> Self {
        Self {
-            policy_docs: value.policy_docs.load(),
-            users: value.users.load(),
-            user_policies: value.user_policies.load(),
-            sts_accounts: value.sts_accounts.load(),
-            sts_policies: value.sts_policies.load(),
-            groups: value.groups.load(),
-            user_group_memberships: value.user_group_memberships.load(),
-            group_policies: value.group_policies.load(),
+            snapshot: value.snapshot(),
        }
    }
 }
@@ -255,98 +431,139 @@ impl From<&Cache> for CacheInner {
 mod tests {
    use std::sync::Arc;

-    use arc_swap::ArcSwap;
    use futures::future::join_all;
+    use rustfs_policy::auth::UserIdentity;
    use time::OffsetDateTime;

-    use super::CacheEntity;
    use crate::cache::Cache;
+    use crate::store::MappedPolicy;

    #[tokio::test]
    async fn test_cache_entity_add() {
-        let cache = ArcSwap::new(Arc::new(CacheEntity::<usize>::default()));
+        let owner = Arc::new(Cache::default());

        let mut f = vec![];

        for (index, key) in (0..100).map(|x| x.to_string()).enumerate() {
-            let c = &cache;
+            let owner = Arc::clone(&owner);
            f.push(async move {
-                Cache::add_or_update(c, &key, &index, OffsetDateTime::now_utc());
+                let user = UserIdentity {
+                    version: index as i64,
+                    ..Default::default()
+                };
+                owner.add_or_update_user(&key, &user, OffsetDateTime::now_utc());
            });
        }
        join_all(f).await;

-        let cache = cache.load();
+        let cache = owner.snapshot();
        for (index, key) in (0..100).map(|x| x.to_string()).enumerate() {
-            assert_eq!(cache.get(&key), Some(&index));
+            assert_eq!(cache.users.get(&key).map(|user| user.version), Some(index as i64));
        }
    }

    #[tokio::test]
    async fn test_cache_entity_update() {
-        let cache = ArcSwap::new(Arc::new(CacheEntity::<usize>::default()));
+        let owner = Arc::new(Cache::default());

        let mut f = vec![];

        for (index, key) in (0..100).map(|x| x.to_string()).enumerate() {
-            let c = &cache;
+            let owner = Arc::clone(&owner);
            f.push(async move {
-                Cache::add_or_update(c, &key, &index, OffsetDateTime::now_utc());
+                let user = UserIdentity {
+                    version: index as i64,
+                    ..Default::default()
+                };
+                owner.add_or_update_user(&key, &user, OffsetDateTime::now_utc());
            });
        }
        join_all(f).await;

-        let cache_load = cache.load();
+        let cache_load = owner.snapshot();
        for (index, key) in (0..100).map(|x| x.to_string()).enumerate() {
-            assert_eq!(cache_load.get(&key), Some(&index));
+            assert_eq!(cache_load.users.get(&key).map(|user| user.version), Some(index as i64));
        }

        let mut f = vec![];

        for (index, key) in (0..100).map(|x| x.to_string()).enumerate() {
-            let c = &cache;
+            let owner = Arc::clone(&owner);
            f.push(async move {
-                Cache::add_or_update(c, &key, &(index * 1000), OffsetDateTime::now_utc());
+                let user = UserIdentity {
+                    version: (index * 1000) as i64,
+                    ..Default::default()
+                };
+                owner.add_or_update_user(&key, &user, OffsetDateTime::now_utc());
            });
        }
        join_all(f).await;

-        let cache_load = cache.load();
+        let cache_load = owner.snapshot();
        for (index, key) in (0..100).map(|x| x.to_string()).enumerate() {
-            assert_eq!(cache_load.get(&key), Some(&(index * 1000)));
+            assert_eq!(cache_load.users.get(&key).map(|user| user.version), Some((index * 1000) as i64));
        }
    }

    #[tokio::test]
    async fn test_cache_entity_delete() {
-        let cache = ArcSwap::new(Arc::new(CacheEntity::<usize>::default()));
+        let owner = Arc::new(Cache::default());

        let mut f = vec![];

        for (index, key) in (0..100).map(|x| x.to_string()).enumerate() {
-            let c = &cache;
+            let owner = Arc::clone(&owner);
            f.push(async move {
-                Cache::add_or_update(c, &key, &index, OffsetDateTime::now_utc());
+                let user = UserIdentity {
+                    version: index as i64,
+                    ..Default::default()
+                };
+                owner.add_or_update_user(&key, &user, OffsetDateTime::now_utc());
            });
        }
        join_all(f).await;

-        let cache_load = cache.load();
+        let cache_load = owner.snapshot();
        for (index, key) in (0..100).map(|x| x.to_string()).enumerate() {
-            assert_eq!(cache_load.get(&key), Some(&index));
+            assert_eq!(cache_load.users.get(&key).map(|user| user.version), Some(index as i64));
        }
+        drop(cache_load);

        let mut f = vec![];

        for key in (0..100).map(|x| x.to_string()) {
-            let c = &cache;
+            let owner = Arc::clone(&owner);
            f.push(async move {
-                Cache::delete(c, &key, OffsetDateTime::now_utc());
+                owner.delete_user(&key, OffsetDateTime::now_utc());
            });
        }
        join_all(f).await;

-        let cache_load = cache.load();
-        assert!(cache_load.is_empty());
+        let cache_load = owner.snapshot();
+        assert!(cache_load.users.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_cache_snapshot_reads_one_published_state() {
+        let cache = Cache::default();
+        let before = cache.snapshot();
+        let user = UserIdentity {
+            version: 7,
+            ..Default::default()
+        };
+        let policy = MappedPolicy::new("readwrite");
+
+        cache.with_write_lock(|cache| {
+            let now = OffsetDateTime::now_utc();
+            cache.add_or_update_user("snapshot-user", &user, now);
+            cache.add_or_update_user_policy("snapshot-user", &policy, now);
+        });
+
+        assert!(!before.users.contains_key("snapshot-user"));
+        assert!(!before.user_policies.contains_key("snapshot-user"));
+
+        let after = cache.snapshot();
+        assert!(after.users.contains_key("snapshot-user"));
+        assert!(after.user_policies.contains_key("snapshot-user"));
    }
 }
@@ -362,6 +362,7 @@ impl ObjectStore {
        let (tx, mut rx) = mpsc::channel::<ObjectInfoOrErr>(100);

        let path = prefix.to_owned();
+        let sender_on_error = sender.clone();
        tokio::spawn(async move {
            if let Err(err) = store
                .walk(ctx.clone(), Self::BUCKET_NAME, &path, tx, WalkOptions::default())
@@ -378,6 +379,12 @@ impl ObjectStore {
                    error = %err,
                    "system path walk failed"
                );
+                let _ = sender_on_error
+                    .send(StringOrErr {
+                        item: None,
+                        err: Some(err.into()),
+                    })
+                    .await;
            }
        });

@@ -1018,6 +1025,7 @@ impl Store for ObjectStore {
    }

    async fn load_all(&self, cache: &Cache) -> Result<()> {
+        let cache_snapshot = cache.snapshot();
        let listed_config_items = self.list_all_iamconfig_items().await?;

        let mut policy_docs_cache = CacheEntity::new(get_default_policyes());
@@ -1059,16 +1067,12 @@ impl Store for ObjectStore {
            }
        }

-        cache.policy_docs.store(Arc::new(policy_docs_cache.update_load_time()));
-
        let mut user_items_cache = CacheEntity::default();

        // users
        if let Some(item_name_list) = listed_config_items.get(USERS_LIST_KEY) {
            let mut item_name_list = item_name_list.clone();

-            // let mut items_cache = CacheEntity::default();
-
            loop {
                if item_name_list.len() < 32 {
                    let items = self.load_user_concurrent(&item_name_list, UserType::Reg).await?;
@@ -1099,11 +1103,10 @@ impl Store for ObjectStore {

                item_name_list = item_name_list.split_off(32);
            }
-
-            // cache.users.store(Arc::new(items_cache.update_load_time()));
        }

        // groups
+        let mut groups_cache = None;
        if let Some(item_name_list) = listed_config_items.get(GROUPS_LIST_KEY) {
            let mut items_cache = CacheEntity::default();

@@ -1115,10 +1118,11 @@ impl Store for ObjectStore {
                };
            }

-            cache.groups.store(Arc::new(items_cache.update_load_time()));
+            groups_cache = Some(items_cache);
        }

        // user policies
+        let mut user_policies_cache = None;
        if let Some(item_name_list) = listed_config_items.get(POLICY_DB_USERS_LIST_KEY) {
            let mut item_name_list = item_name_list.clone();

@@ -1159,10 +1163,11 @@ impl Store for ObjectStore {
                item_name_list = item_name_list.split_off(32);
            }

-            cache.user_policies.store(Arc::new(items_cache.update_load_time()));
+            user_policies_cache = Some(items_cache);
        }

        // group policy
+        let mut group_policies_cache = None;
        if let Some(item_name_list) = listed_config_items.get(POLICY_DB_GROUPS_LIST_KEY) {
            let mut items_cache = CacheEntity::default();

@@ -1177,7 +1182,7 @@ impl Store for ObjectStore {
                };
            }

-            cache.group_policies.store(Arc::new(items_cache.update_load_time()));
+            group_policies_cache = Some(items_cache);
        }

        let mut sts_policies_cache = CacheEntity::default();
@@ -1212,11 +1217,8 @@ impl Store for ObjectStore {

            // Merge items_cache to user_items_cache
            user_items_cache.extend(items_cache);
-
-            // cache.users.store(Arc::new(items_cache.update_load_time()));
        }

-        cache.build_user_group_memberships();
        let mut sts_items_cache = CacheEntity::default();
        // sts users
        if let Some(item_name_list) = listed_config_items.get(STS_LIST_KEY) {
@@ -1245,159 +1247,29 @@ impl Store for ObjectStore {
            }
        }

-        cache.users.store(Arc::new(user_items_cache.update_load_time()));
-        cache.sts_accounts.store(Arc::new(sts_items_cache.update_load_time()));
-        cache.sts_policies.store(Arc::new(sts_policies_cache.update_load_time()));
+        cache.with_write_lock(|cache| {
+            if cache.matches_snapshot(&cache_snapshot) {
+                cache.replace_policy_docs(policy_docs_cache);
+                if let Some(groups_cache) = groups_cache {
+                    cache.replace_groups(groups_cache);
+                }
+                if let Some(user_policies_cache) = user_policies_cache {
+                    cache.replace_user_policies(user_policies_cache);
+                }
+                if let Some(group_policies_cache) = group_policies_cache {
+                    cache.replace_group_policies(group_policies_cache);
+                }
+                cache.replace_users(user_items_cache);
+                cache.replace_sts_accounts(sts_items_cache);
+                cache.replace_sts_policies(sts_policies_cache);
+                cache.build_user_group_memberships();
+            } else {
+                warn!("skip IAM full reload cache commit because one or more IAM caches changed during reload");
+            }
+        });

        Ok(())
    }
-
-    // /// load all and make a new cache.
-    // async fn load_all(&self, cache: &Cache) -> Result<()> {
-    //     let _items = &[
-    //         "policydb/",
-    //         "policies/",
-    //         "groups/",
-    //         "policydb/users/",
-    //         "policydb/groups/",
-    //         "service-accounts/",
-    //         "policydb/sts-users/",
-    //         "sts/",
-    //     ];
-
-    //     let items = self.list_iam_config_items("config/iam/").await?;
-    //     debug!("all iam items: {items:?}");
-
-    //     let (policy_docs, users, user_policies, sts_policies, sts_accounts) = (
-    //         Arc::new(tokio::sync::Mutex::new(CacheEntity::new(Self::get_default_policyes()))),
-    //         Arc::new(tokio::sync::Mutex::new(CacheEntity::default())),
-    //         Arc::new(tokio::sync::Mutex::new(CacheEntity::default())),
-    //         Arc::new(tokio::sync::Mutex::new(CacheEntity::default())),
-    //         Arc::new(tokio::sync::Mutex::new(CacheEntity::default())),
-    //     );
-
-    //     // Read 32 elements at a time
-    //     let iter = items
-    //         .iter()
-    //         .map(|item| item.trim_start_matches("config/iam/"))
-    //         .map(|item| split_path(item, item.starts_with("policydb/")))
-    //         .filter_map(|(list_key, trimmed_item)| {
-    //             debug!("list_key: {list_key}, trimmed_item: {trimmed_item}");
-
-    //             if list_key == "format.json" {
-    //                 return None;
-    //             }
-
-    //             let (policy_docs, users, user_policies, sts_policies, sts_accounts) = (
-    //                 policy_docs.clone(),
-    //                 users.clone(),
-    //                 user_policies.clone(),
-    //                 sts_policies.clone(),
-    //                 sts_accounts.clone(),
-    //             );
-
-    //             Some(async move {
-    //                 match list_key {
-    //                     "policies/" => {
-    //                         let trimmed_item = dir(trimmed_item);
-    //                         let name = trimmed_item.trim_end_matches('/');
-    //                         let policy_doc = self.load_policy(name).await?;
-    //                         policy_docs.lock().await.insert(name.to_owned(), policy_doc);
-    //                     }
-    //                     "users/" => {
-    //                         let name = dir(trimmed_item);
-    //                         if let Some(user) = self.load_user_identity(UserType::Reg, &name).await? {
-    //                             users.lock().await.insert(name.to_owned(), user);
-    //                         };
-    //                     }
-    //                     "groups/" => {}
-    //                     "policydb/users/" | "policydb/groups/" => {
-    //                         let name = trimmed_item.strip_suffix(".json").unwrap_or(trimmed_item);
-    //                         let mapped_policy = self
-    //                             .load_mapped_policy(UserType::Reg, name, list_key == "policydb/groups/")
-    //                             .await?;
-    //                         if !mapped_policy.policies.is_empty() {
-    //                             user_policies.lock().await.insert(name.to_owned(), mapped_policy);
-    //                         }
-    //                     }
-    //                     "service-accounts/" => {
-    //                         let trimmed_item = dir(trimmed_item);
-    //                         let name = trimmed_item.trim_end_matches('/');
-    //                         let Some(user) = self.load_user_identity(UserType::Svc, name).await? else {
-    //                             return Ok(());
-    //                         };
-
-    //                         let parent = user.credentials.parent_user.clone();
-
-    //                         {
-    //                             users.lock().await.insert(name.to_owned(), user);
-    //                         }
-
-    //                         if users.lock().await.get(&parent).is_some() {
-    //                             return Ok(());
-    //                         }
-
-    //                         match self.load_mapped_policy(UserType::Sts, parent.as_str(), false).await {
-    //                             Ok(m) => sts_policies.lock().await.insert(name.to_owned(), m),
-    //                             Err(Error::EcstoreError(e)) if is_err_config_not_found(&e) => return Ok(()),
-    //                             Err(e) => return Err(e),
-    //                         };
-    //                     }
-    //                     "sts/" => {
-    //                         let name = dir(trimmed_item);
-    //                         if let Some(user) = self.load_user_identity(UserType::Sts, &name).await? {
-    //                             warn!("sts_accounts insert {}, user {:?}", name, &user.credentials.access_key);
-    //                             sts_accounts.lock().await.insert(name.to_owned(), user);
-    //                         };
-    //                     }
-    //                     "policydb/sts-users/" => {
-    //                         let name = trimmed_item.strip_suffix(".json").unwrap_or(trimmed_item);
-    //                         let mapped_policy = self.load_mapped_policy(UserType::Sts, name, false).await?;
-    //                         if !mapped_policy.policies.is_empty() {
-    //                             sts_policies.lock().await.insert(name.to_owned(), mapped_policy);
-    //                         }
-    //                     }
-    //                     _ => {}
-    //                 }
-
-    //                 Result::Ok(())
-    //             })
-    //         });
-
-    //     let mut all_futures = Vec::with_capacity(32);
-
-    //     for f in iter {
-    //         all_futures.push(f);
-
-    //         if all_futures.len() == 32 {
-    //             try_join_all(all_futures).await?;
-    //             all_futures = Vec::with_capacity(32);
-    //         }
-    //     }
-
-    //     if !all_futures.is_empty() {
-    //         try_join_all(all_futures).await?;
-    //     }
-
-    //     if let Some(x) = Arc::into_inner(users) {
-    //         cache.users.store(Arc::new(x.into_inner().update_load_time()))
-    //     }
-
-    //     if let Some(x) = Arc::into_inner(policy_docs) {
-    //         cache.policy_docs.store(Arc::new(x.into_inner().update_load_time()))
-    //     }
-    //     if let Some(x) = Arc::into_inner(user_policies) {
-    //         cache.user_policies.store(Arc::new(x.into_inner().update_load_time()))
-    //     }
-    //     if let Some(x) = Arc::into_inner(sts_policies) {
-    //         cache.sts_policies.store(Arc::new(x.into_inner().update_load_time()))
-    //     }
-    //     if let Some(x) = Arc::into_inner(sts_accounts) {
-    //         cache.sts_accounts.store(Arc::new(x.into_inner().update_load_time()))
-    //     }
-
-    //     Ok(())
-    // }
 }

 #[cfg(test)]
@@ -1325,7 +1325,7 @@ mod tests {
    use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
    use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
    use serde_json::Value;
-    use std::collections::HashMap;
+    use std::collections::{HashMap, HashSet};
    use time::OffsetDateTime;

    #[test]
@@ -1410,6 +1410,10 @@ mod tests {
        }

        async fn load_user(&self, name: &str, user_type: UserType, m: &mut HashMap<String, UserIdentity>) -> Result<()> {
+            if name == "deleted-notify-user" {
+                return Err(Error::NoSuchUser(name.to_string()));
+            }
+
            if user_type == UserType::Reg && name == "load-failure-user" {
                return Err(Error::Io(std::io::Error::other("load user failed")));
            }
@@ -1442,7 +1446,10 @@ mod tests {
            Err(Error::InvalidArgument)
        }

-        async fn load_group(&self, _name: &str, _m: &mut HashMap<String, GroupInfo>) -> Result<()> {
+        async fn load_group(&self, name: &str, m: &mut HashMap<String, GroupInfo>) -> Result<()> {
+            if name == "notify-group" {
+                m.insert(name.to_string(), GroupInfo::new(vec!["notify-user".to_string()]));
+            }
            Ok(())
        }

@@ -1495,6 +1502,9 @@ mod tests {
            if user_type == UserType::Reg && !is_group && name == "notify-user" {
                m.insert(name.to_string(), MappedPolicy::new("readwrite"));
            }
+            if user_type == UserType::Sts && !is_group && name == "notify-sts-parent" {
+                m.insert(name.to_string(), MappedPolicy::new("readwrite"));
+            }
            Ok(())
        }

@@ -1512,9 +1522,6 @@ mod tests {
            let custom_claim_policy =
                Policy::parse_config(CUSTOM_STS_CLAIM_POLICY_JSON.as_bytes()).expect("custom STS claim policy should parse");
            policy_docs.insert(CUSTOM_STS_CLAIM_POLICY.to_string(), PolicyDoc::new(custom_claim_policy));
-            cache
-                .policy_docs
-                .store(Arc::new(CacheEntity::new(policy_docs).update_load_time()));

            if self.empty_policies {
                const PARENT_USER: &str = "sts-empty-parent-policy-test";
@@ -1537,16 +1544,17 @@ mod tests {
                };
                let mut users = HashMap::new();
                users.insert(PARENT_USER.to_string(), parent_identity);
-                cache.users.store(Arc::new(CacheEntity::new(users).update_load_time()));

-                cache.groups.store(Arc::new(CacheEntity::default().update_load_time()));
-                cache
-                    .group_policies
-                    .store(Arc::new(CacheEntity::default().update_load_time()));
-                cache.user_policies.store(Arc::new(CacheEntity::default().update_load_time()));
-                cache.sts_accounts.store(Arc::new(CacheEntity::default().update_load_time()));
-                cache.sts_policies.store(Arc::new(CacheEntity::default().update_load_time()));
-                cache.build_user_group_memberships();
+                cache.with_write_lock(|cache| {
+                    cache.replace_policy_docs(CacheEntity::new(policy_docs));
+                    cache.replace_users(CacheEntity::new(users));
+                    cache.replace_groups(CacheEntity::default());
+                    cache.replace_group_policies(CacheEntity::default());
+                    cache.replace_user_policies(CacheEntity::default());
+                    cache.replace_sts_accounts(CacheEntity::default());
+                    cache.replace_sts_policies(CacheEntity::default());
+                    cache.build_user_group_memberships();
+                });
                return Ok(());
            }

@@ -1572,24 +1580,25 @@ mod tests {
            };
            let mut users = HashMap::new();
            users.insert(PARENT_USER.to_string(), parent_identity);
-            cache.users.store(Arc::new(CacheEntity::new(users).update_load_time()));

            let group = GroupInfo::new(vec![PARENT_USER.to_string()]);
            let mut groups = HashMap::new();
            groups.insert(GROUP_NAME.to_string(), group);
-            cache.groups.store(Arc::new(CacheEntity::new(groups).update_load_time()));

            let group_policy = MappedPolicy::new("readwrite");
            let mut group_policies = HashMap::new();
            group_policies.insert(GROUP_NAME.to_string(), group_policy);
-            cache
-                .group_policies
-                .store(Arc::new(CacheEntity::new(group_policies).update_load_time()));

-            cache.user_policies.store(Arc::new(CacheEntity::default().update_load_time()));
-            cache.sts_accounts.store(Arc::new(CacheEntity::default().update_load_time()));
-            cache.sts_policies.store(Arc::new(CacheEntity::default().update_load_time()));
-            cache.build_user_group_memberships();
+            cache.with_write_lock(|cache| {
+                cache.replace_policy_docs(CacheEntity::new(policy_docs));
+                cache.replace_users(CacheEntity::new(users));
+                cache.replace_groups(CacheEntity::new(groups));
+                cache.replace_group_policies(CacheEntity::new(group_policies));
+                cache.replace_user_policies(CacheEntity::default());
+                cache.replace_sts_accounts(CacheEntity::default());
+                cache.replace_sts_policies(CacheEntity::default());
+                cache.build_user_group_memberships();
+            });

            Ok(())
        }
@@ -1950,7 +1959,10 @@ mod tests {
            parent_user: parent_user.to_string(),
            ..Default::default()
        });
-        Cache::add_or_update(&iam_sys.store.cache.sts_accounts, sts_access_key, &sts_user, OffsetDateTime::now_utc());
+        iam_sys
+            .store
+            .cache
+            .add_or_update_sts_account(sts_access_key, &sts_user, OffsetDateTime::now_utc());

        let mut claims = HashMap::new();
        claims.insert(POLICYNAME.to_string(), Value::String(CUSTOM_STS_CLAIM_POLICY.to_string()));
@@ -1991,7 +2003,10 @@ mod tests {
            parent_user: parent_user.to_string(),
            ..Default::default()
        });
-        Cache::add_or_update(&iam_sys.store.cache.sts_accounts, sts_access_key, &sts_user, OffsetDateTime::now_utc());
+        iam_sys
+            .store
+            .cache
+            .add_or_update_sts_account(sts_access_key, &sts_user, OffsetDateTime::now_utc());

        let mut claims = HashMap::new();
        claims.insert(
@@ -2035,7 +2050,10 @@ mod tests {
            parent_user: parent_user.to_string(),
            ..Default::default()
        });
-        Cache::add_or_update(&iam_sys.store.cache.sts_accounts, sts_access_key, &sts_user, OffsetDateTime::now_utc());
+        iam_sys
+            .store
+            .cache
+            .add_or_update_sts_account(sts_access_key, &sts_user, OffsetDateTime::now_utc());

        let mut claims = HashMap::new();
        claims.insert(POLICYNAME.to_string(), Value::String(CUSTOM_STS_CLAIM_POLICY.to_string()));
@@ -2227,6 +2245,113 @@ mod tests {
        );
    }

+    #[tokio::test]
+    async fn test_group_notification_populates_new_membership_entry() {
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await.unwrap();
+        let iam_sys = IamSys::new(cache_manager);
+
+        iam_sys.store.cache.with_write_lock(|cache| {
+            cache.replace_user_group_memberships(CacheEntity::default());
+        });
+
+        iam_sys.load_group("notify-group").await.unwrap();
+
+        let cache = iam_sys.store.cache.snapshot();
+        let memberships = &cache.user_group_memberships;
+        assert!(
+            memberships
+                .get("notify-user")
+                .is_some_and(|groups| groups.contains("notify-group")),
+            "group notification must create a reverse membership entry for first-time members"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_sts_policy_mapping_notification_updates_sts_policy_cache() {
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await.unwrap();
+        let iam_sys = IamSys::new(cache_manager);
+
+        iam_sys
+            .load_policy_mapping("notify-sts-parent", UserType::Sts, false)
+            .await
+            .unwrap();
+
+        let cache = iam_sys.store.cache.snapshot();
+        let sts_policies = &cache.sts_policies;
+        assert!(
+            sts_policies.contains_key("notify-sts-parent"),
+            "STS policy mapping notifications must update sts_policies instead of deleting them"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_missing_user_notification_cleans_related_cache_state() {
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await.unwrap();
+        let iam_sys = IamSys::new(cache_manager);
+
+        const USER: &str = "deleted-notify-user";
+        const GROUP: &str = "deleted-notify-group";
+        const SVC_CHILD: &str = "deleted-notify-service-child";
+        const STS_CHILD: &str = "deleted-notify-sts-child";
+        const OTHER_USER: &str = "deleted-notify-other-user";
+
+        let user = UserIdentity::from(Credentials {
+            access_key: USER.to_string(),
+            secret_key: "longenoughsecret".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+
+        let mut service_claims = HashMap::new();
+        service_claims.insert(iam_policy_claim_name_sa(), Value::String(INHERITED_POLICY_TYPE.to_string()));
+        let service_child = UserIdentity::from(Credentials {
+            access_key: SVC_CHILD.to_string(),
+            secret_key: "longenoughsecret".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            parent_user: USER.to_string(),
+            claims: Some(service_claims),
+            ..Default::default()
+        });
+
+        let sts_child = UserIdentity::from(Credentials {
+            access_key: STS_CHILD.to_string(),
+            secret_key: "longenoughsecret".to_string(),
+            session_token: "session-token".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            parent_user: USER.to_string(),
+            ..Default::default()
+        });
+
+        let membership = HashSet::from([GROUP.to_string()]);
+        let group = GroupInfo::new(vec![USER.to_string(), OTHER_USER.to_string()]);
+        let mapped_policy = MappedPolicy::new("readwrite");
+
+        iam_sys.store.cache.with_write_lock(|cache| {
+            let now = OffsetDateTime::now_utc();
+            cache.add_or_update_user(USER, &user, now);
+            cache.add_or_update_user_policy(USER, &mapped_policy, now);
+            cache.add_or_update_group(GROUP, &group, now);
+            cache.add_or_update_user_group_membership(USER, &membership, now);
+            cache.add_or_update_user(SVC_CHILD, &service_child, now);
+            cache.add_or_update_sts_account(STS_CHILD, &sts_child, now);
+        });
+
+        iam_sys.load_user(USER, UserType::Reg).await.unwrap();
+
+        let cache = iam_sys.store.cache.snapshot();
+        assert!(!cache.users.contains_key(USER));
+        assert!(!cache.user_policies.contains_key(USER));
+        assert!(!cache.user_group_memberships.contains_key(USER));
+        assert!(!cache.users.contains_key(SVC_CHILD));
+        assert!(!cache.sts_accounts.contains_key(STS_CHILD));
+        let group = cache.groups.get(GROUP).expect("group should remain after member removal");
+        assert!(!group.members.contains(&USER.to_string()));
+        assert!(group.members.contains(&OTHER_USER.to_string()));
+    }
+
    #[tokio::test]
    async fn test_check_key_propagates_cache_miss_load_failure() {
        let store = StsTestMockStore { empty_policies: false };
@@ -2281,7 +2406,10 @@ mod tests {
            parent_user: "sts-empty-parent-policy-test".to_string(),
            ..Default::default()
        });
-        Cache::add_or_update(&iam_sys.store.cache.sts_accounts, sts_access_key, &sts_user, OffsetDateTime::now_utc());
+        iam_sys
+            .store
+            .cache
+            .add_or_update_sts_account(sts_access_key, &sts_user, OffsetDateTime::now_utc());

        let mut claims = HashMap::new();
        claims.insert(
@@ -2395,7 +2523,10 @@ mod tests {
            parent_user: "sts-empty-parent-policy-test".to_string(),
            ..Default::default()
        });
-        Cache::add_or_update(&iam_sys.store.cache.sts_accounts, sts_access_key, &sts_user, OffsetDateTime::now_utc());
+        iam_sys
+            .store
+            .cache
+            .add_or_update_sts_account(sts_access_key, &sts_user, OffsetDateTime::now_utc());

        let session_policy_json = r#"{
  "Version":"2012-10-17",
@@ -2445,12 +2576,10 @@ mod tests {
            claims: Some(service_account_claims),
            ..Default::default()
        });
-        Cache::add_or_update(
-            &iam_sys.store.cache.users,
-            service_account_access_key,
-            &service_identity,
-            OffsetDateTime::now_utc(),
-        );
+        iam_sys
+            .store
+            .cache
+            .add_or_update_user(service_account_access_key, &service_identity, OffsetDateTime::now_utc());

        let mut request_claims = HashMap::new();
        request_claims.insert("parent".to_string(), Value::String(parent_user.to_string()));
@@ -30,6 +30,18 @@ use uuid::Uuid;

 const UNLOCK_RETRY_ATTEMPTS: usize = 3;
 const UNLOCK_RETRY_BACKOFF: Duration = Duration::from_millis(100);
+const LOCK_ACQUIRE_RETRY_ATTEMPTS: usize = 3;
+const LOCK_ACQUIRE_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
+const REMOTE_LOCK_RPC_FAILED_PREFIX: &str = "remote lock rpc failed:";
+const REMOTE_LOCK_RPC_TIMED_OUT_PREFIX: &str = "remote lock rpc timed out:";
+const UNRECOVERABLE_QUORUM_FAILURE_PREFIX: &str = "unrecoverable quorum failure";
+
+#[derive(Debug)]
+enum LockAcquireFailureKind {
+    NonRetryable,
+    RetryableContention,
+    UnrecoverableQuorum,
+}

 /// Generate a new aggregate lock ID for multiple client locks
 fn generate_aggregate_lock_id(resource: &ObjectKey) -> LockId {
@@ -39,6 +51,35 @@ fn generate_aggregate_lock_id(resource: &ObjectKey) -> LockId {
    }
 }

+fn is_remote_lock_rpc_failure(error: &str) -> bool {
+    has_case_insensitive_prefix(error, REMOTE_LOCK_RPC_FAILED_PREFIX)
+        || has_case_insensitive_prefix(error, REMOTE_LOCK_RPC_TIMED_OUT_PREFIX)
+}
+
+fn has_case_insensitive_prefix(error: &str, expected_prefix: &str) -> bool {
+    error
+        .get(0..expected_prefix.len())
+        .is_some_and(|prefix| prefix.eq_ignore_ascii_case(expected_prefix))
+}
+
+fn is_unrecoverable_quorum_error(error: &str) -> bool {
+    error
+        .get(0..UNRECOVERABLE_QUORUM_FAILURE_PREFIX.len())
+        .is_some_and(|prefix| prefix.eq_ignore_ascii_case(UNRECOVERABLE_QUORUM_FAILURE_PREFIX))
+}
+
+fn classify_lock_failure(error: &str) -> LockAcquireFailureKind {
+    if is_unrecoverable_quorum_error(error) {
+        return LockAcquireFailureKind::UnrecoverableQuorum;
+    }
+
+    if error.to_ascii_lowercase().contains("timeout") || is_remote_lock_rpc_failure(error) {
+        return LockAcquireFailureKind::RetryableContention;
+    }
+
+    LockAcquireFailureKind::NonRetryable
+}
+
 /// A RAII guard for distributed locks that releases the lock asynchronously when dropped.
 #[derive(Debug)]
 pub struct DistributedLockGuard {
@@ -131,6 +172,12 @@ pub struct DistributedLock {

 type LockAcquireTaskResult = (usize, Result<LockResponse>);

+struct LockAcquireQuorumResult {
+    response: LockResponse,
+    individual_locks: Vec<(LockId, Arc<dyn LockClient>)>,
+    failure_kind: Option<LockAcquireFailureKind>,
+}
+
 impl DistributedLock {
    /// Create new distributed lock
    pub fn new(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
@@ -185,7 +232,12 @@ impl DistributedLock {
        }

        let required_quorum = self.required_quorum(request.lock_type);
-        let (resp, individual_locks) = self.acquire_lock_quorum(request).await?;
+        let LockAcquireQuorumResult {
+            response: resp,
+            individual_locks,
+            failure_kind,
+            ..
+        } = self.acquire_lock_quorum_with_retry(request).await?;
        if resp.success {
            // Use aggregate lock_id from LockResponse's LockInfo
            // The aggregate id is what we expose to callers; individual_locks carries
@@ -207,26 +259,36 @@ impl DistributedLock {
                        "acquire_lock_quorum contention: {}",
                        error_msg
                    );
-                } else {
+                } else if matches!(failure_kind, Some(LockAcquireFailureKind::NonRetryable)) {
                    warn!(
                        resource = %request.resource,
                        owner = %request.owner,
                        "acquire_lock_quorum error: {}",
                        error_msg
                    );
+                } else if matches!(failure_kind, Some(LockAcquireFailureKind::RetryableContention)) {
+                    debug!(
+                        resource = %request.resource,
+                        owner = %request.owner,
+                        "acquire_lock_quorum contention: {}",
+                        error_msg
+                    );
+                } else {
+                    debug!(
+                        resource = %request.resource,
+                        owner = %request.owner,
+                        "acquire_lock_quorum final failure: {}",
+                        error_msg
+                    );
                }
-                if error_msg.contains("quorum") {
-                    // This is a quorum failure - return appropriate error
+
+                if matches!(failure_kind, Some(LockAcquireFailureKind::UnrecoverableQuorum)) {
                    let achieved = individual_locks.len();
                    Err(LockError::QuorumNotReached {
                        required: required_quorum,
                        achieved,
                    })
-                } else if error_msg.contains("timeout") || resp.wait_time >= request.acquire_timeout {
-                    // This is a timeout - return None so caller can convert to timeout error
-                    Ok(None)
                } else {
-                    // Other failure - return None for backward compatibility
                    Ok(None)
                }
            } else {
@@ -287,6 +349,16 @@ impl DistributedLock {
        pending
    }

+    fn lock_acquire_retry_backoff(attempt: usize) -> Duration {
+        LOCK_ACQUIRE_RETRY_INITIAL_BACKOFF * attempt as u32
+    }
+
+    fn is_retryable_lock_failure(resp: &LockResponse) -> bool {
+        resp.error
+            .as_deref()
+            .is_some_and(|error| matches!(classify_lock_failure(error), LockAcquireFailureKind::RetryableContention))
+    }
+
    async fn release_entries(entries: Vec<(LockId, Arc<dyn LockClient>)>, context: &'static str) {
        let mut pending = entries;

@@ -408,14 +480,58 @@ impl DistributedLock {
        }
    }

+    async fn acquire_lock_quorum_with_retry(&self, request: &LockRequest) -> Result<LockAcquireQuorumResult> {
+        let start = std::time::Instant::now();
+        let mut attempt = 1;
+        let mut last_result = None;
+
+        loop {
+            let elapsed = start.elapsed();
+            if elapsed >= request.acquire_timeout {
+                break;
+            }
+
+            let remaining = request.acquire_timeout - elapsed;
+            let mut attempt_request = request.clone();
+            attempt_request.acquire_timeout = remaining;
+            attempt_request.lock_id = LockId::new_unique(&request.resource);
+
+            let result = self.acquire_lock_quorum_once(&attempt_request).await?;
+            if result.response.success
+                || !result.individual_locks.is_empty()
+                || !Self::is_retryable_lock_failure(&result.response)
+                || attempt >= LOCK_ACQUIRE_RETRY_ATTEMPTS
+            {
+                return Ok(result);
+            }
+
+            last_result = Some(result);
+            let backoff = Self::lock_acquire_retry_backoff(attempt);
+            if start.elapsed().saturating_add(backoff) >= request.acquire_timeout {
+                break;
+            }
+
+            tokio::time::sleep(backoff).await;
+            attempt += 1;
+        }
+
+        Ok(last_result.unwrap_or_else(|| LockAcquireQuorumResult {
+            response: LockResponse::failure("Lock acquisition timeout", request.acquire_timeout),
+            individual_locks: Vec::new(),
+            failure_kind: Some(LockAcquireFailureKind::RetryableContention),
+        }))
+    }
+
    /// Quorum-based lock acquisition: success if at least the required quorum succeeds.
    /// Collects all individual lock_ids from successful clients and creates an aggregate lock_id.
    /// Returns the LockResponse with aggregate lock_id and individual lock mappings.
-    async fn acquire_lock_quorum(&self, request: &LockRequest) -> Result<(LockResponse, Vec<(LockId, Arc<dyn LockClient>)>)> {
+    async fn acquire_lock_quorum_once(&self, request: &LockRequest) -> Result<LockAcquireQuorumResult> {
        let required_quorum = self.required_quorum(request.lock_type);
        let mut pending = self.spawn_lock_requests(request);
        let mut individual_locks: Vec<(LockId, Arc<dyn LockClient>)> = Vec::new();
        let fallback_lock_id = request.lock_id.clone();
+        let mut last_failure = None;
+        let mut hard_failures = 0usize;

        while let Some(join_result) = pending.join_next().await {
            match join_result {
@@ -434,17 +550,52 @@ impl DistributedLock {
                        }
                    } else {
                        let error = resp.error.unwrap_or_else(|| "unknown error".to_string());
-                        self.log_failed_lock_response(request, idx, error);
+                        if is_remote_lock_rpc_failure(&error) {
+                            hard_failures += 1;
+                        }
+                        self.log_failed_lock_response(request, idx, error.clone());
+                        last_failure = Some(error);
                    }
                }
                Ok((idx, Err(err))) => {
+                    hard_failures += 1;
                    tracing::warn!("Failed to acquire lock on client {}: {}", idx, err);
+                    last_failure = Some(err.to_string());
                }
                Err(err) => {
+                    hard_failures += 1;
                    tracing::warn!("Lock acquisition task join failed: {}", err);
+                    last_failure = Some(err.to_string());
                }
            }

+            if self.clients.len().saturating_sub(hard_failures) < required_quorum {
+                let rollback_count = individual_locks.len();
+                Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
+                if !pending.is_empty() {
+                    Self::spawn_pending_cleanup(
+                        pending,
+                        self.clients.clone(),
+                        fallback_lock_id.clone(),
+                        "distributed_lock_failure_cleanup",
+                    );
+                }
+
+                let mut error = format!(
+                    "Unrecoverable quorum failure: {rollback_count}/{required_quorum} required; {hard_failures} clients failed"
+                );
+                if let Some(last_failure) = last_failure {
+                    error.push_str("; last failure: ");
+                    error.push_str(&last_failure);
+                }
+                let resp = LockResponse::failure(error, Duration::ZERO);
+                return Ok(LockAcquireQuorumResult {
+                    response: resp,
+                    individual_locks,
+                    failure_kind: Some(LockAcquireFailureKind::UnrecoverableQuorum),
+                });
+            }
+
            if individual_locks.len() >= required_quorum {
                if !pending.is_empty() {
                    Self::spawn_pending_cleanup(
@@ -479,7 +630,11 @@ impl DistributedLock {
                    },
                    Duration::ZERO,
                );
-                return Ok((resp, individual_locks));
+                return Ok(LockAcquireQuorumResult {
+                    response: resp,
+                    individual_locks,
+                    failure_kind: None,
+                });
            }

            if individual_locks.len() + pending.len() < required_quorum {
@@ -494,21 +649,46 @@ impl DistributedLock {
                    );
                }

-                let resp = LockResponse::failure(
-                    format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required"),
-                    Duration::ZERO,
-                );
-                return Ok((resp, individual_locks));
+                let mut error = format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required");
+                let failure_kind = if hard_failures > 0 {
+                    error = format!(
+                        "Unrecoverable quorum failure: {rollback_count}/{required_quorum} required; {hard_failures} clients failed; {error}"
+                    );
+                    LockAcquireFailureKind::UnrecoverableQuorum
+                } else {
+                    LockAcquireFailureKind::RetryableContention
+                };
+                if let Some(last_failure) = last_failure {
+                    error.push_str("; last failure: ");
+                    error.push_str(&last_failure);
+                }
+                let resp = LockResponse::failure(error, Duration::ZERO);
+                return Ok(LockAcquireQuorumResult {
+                    response: resp,
+                    individual_locks,
+                    failure_kind: Some(failure_kind),
+                });
            }
        }

        let rollback_count = individual_locks.len();
        Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
-        let resp = LockResponse::failure(
-            format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required"),
-            Duration::ZERO,
-        );
-        Ok((resp, individual_locks))
+        let mut error = format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required");
+        if let Some(last_failure) = &last_failure {
+            error.push_str("; last failure: ");
+            error.push_str(last_failure);
+        }
+        let resp = LockResponse::failure(error, Duration::ZERO);
+        Ok(LockAcquireQuorumResult {
+            response: resp,
+            individual_locks,
+            failure_kind: Some(if hard_failures > 0 {
+                LockAcquireFailureKind::UnrecoverableQuorum
+            } else {
+                let fallback_kind = last_failure.as_deref().map(classify_lock_failure);
+                fallback_kind.unwrap_or(LockAcquireFailureKind::RetryableContention)
+            }),
+        })
    }
 }

@@ -527,3 +707,442 @@ fn record_lock_held_release(lock_type: LockType) {
        LockType::Exclusive => record_write_lock_held_release(),
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{DistributedLock, is_remote_lock_rpc_failure};
+    use crate::{LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockType, ObjectKey, client::LockClient};
+    use std::{
+        collections::{HashMap, VecDeque},
+        sync::{Arc, Mutex},
+        time::Duration,
+    };
+
+    #[derive(Debug)]
+    struct ResponseClient {
+        response: LockResponse,
+        delay: Duration,
+    }
+
+    impl ResponseClient {
+        fn new(response: LockResponse) -> Self {
+            Self {
+                response,
+                delay: Duration::ZERO,
+            }
+        }
+
+        fn with_delay(mut self, delay: Duration) -> Self {
+            self.delay = delay;
+            self
+        }
+
+        fn into_client(self) -> Arc<dyn LockClient> {
+            Arc::new(self)
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl LockClient for ResponseClient {
+        async fn acquire_lock(&self, _request: &LockRequest) -> crate::Result<LockResponse> {
+            if !self.delay.is_zero() {
+                tokio::time::sleep(self.delay).await;
+            }
+            Ok(self.response.clone())
+        }
+
+        async fn release(&self, _lock_id: &LockId) -> crate::Result<bool> {
+            Ok(false)
+        }
+
+        async fn refresh(&self, _lock_id: &LockId) -> crate::Result<bool> {
+            Ok(false)
+        }
+
+        async fn force_release(&self, _lock_id: &LockId) -> crate::Result<bool> {
+            Ok(false)
+        }
+
+        async fn check_status(&self, _lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
+            Ok(None)
+        }
+
+        async fn get_stats(&self) -> crate::Result<LockStats> {
+            Ok(LockStats::default())
+        }
+
+        async fn close(&self) -> crate::Result<()> {
+            Ok(())
+        }
+
+        async fn is_online(&self) -> bool {
+            true
+        }
+
+        async fn is_local(&self) -> bool {
+            false
+        }
+    }
+
+    #[derive(Debug, Clone, Copy)]
+    enum AcquirePlan {
+        Success { delay: Duration },
+        Failure { error: &'static str, delay: Duration },
+    }
+
+    #[derive(Debug)]
+    struct SequencedClient {
+        plans: Mutex<VecDeque<AcquirePlan>>,
+        active: tokio::sync::Mutex<HashMap<LockId, LockInfo>>,
+        seen_ids: Arc<Mutex<Vec<LockId>>>,
+    }
+
+    impl SequencedClient {
+        fn new(plans: Vec<AcquirePlan>, seen_ids: Arc<Mutex<Vec<LockId>>>) -> Self {
+            Self {
+                plans: Mutex::new(plans.into()),
+                active: tokio::sync::Mutex::new(HashMap::new()),
+                seen_ids,
+            }
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl LockClient for SequencedClient {
+        async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
+            self.seen_ids.lock().unwrap().push(request.lock_id.clone());
+
+            let plan = self
+                .plans
+                .lock()
+                .unwrap()
+                .pop_front()
+                .unwrap_or(AcquirePlan::Success { delay: Duration::ZERO });
+
+            match plan {
+                AcquirePlan::Success { delay } => {
+                    if !delay.is_zero() {
+                        tokio::time::sleep(delay).await;
+                    }
+
+                    let lock_info = LockInfo {
+                        id: request.lock_id.clone(),
+                        resource: request.resource.clone(),
+                        lock_type: request.lock_type,
+                        status: crate::types::LockStatus::Acquired,
+                        owner: request.owner.clone(),
+                        acquired_at: std::time::SystemTime::now(),
+                        expires_at: std::time::SystemTime::now() + request.ttl,
+                        last_refreshed: std::time::SystemTime::now(),
+                        metadata: request.metadata.clone(),
+                        priority: request.priority,
+                        wait_start_time: None,
+                    };
+
+                    self.active.lock().await.insert(request.lock_id.clone(), lock_info.clone());
+
+                    Ok(LockResponse::success(lock_info, Duration::ZERO))
+                }
+                AcquirePlan::Failure { error, delay } => {
+                    if !delay.is_zero() {
+                        tokio::time::sleep(delay).await;
+                    }
+                    Ok(LockResponse::failure(error, Duration::ZERO))
+                }
+            }
+        }
+
+        async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
+            Ok(self.active.lock().await.remove(lock_id).is_some())
+        }
+
+        async fn refresh(&self, _lock_id: &LockId) -> crate::Result<bool> {
+            Ok(false)
+        }
+
+        async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
+            self.release(lock_id).await
+        }
+
+        async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
+            Ok(self.active.lock().await.get(lock_id).cloned())
+        }
+
+        async fn get_stats(&self) -> crate::Result<LockStats> {
+            Ok(LockStats::default())
+        }
+
+        async fn close(&self) -> crate::Result<()> {
+            Ok(())
+        }
+
+        async fn is_online(&self) -> bool {
+            true
+        }
+
+        async fn is_local(&self) -> bool {
+            false
+        }
+    }
+
+    #[test]
+    fn test_is_remote_lock_rpc_failure() {
+        assert!(is_remote_lock_rpc_failure("Remote lock RPC failed: backend unreachable"));
+        assert!(is_remote_lock_rpc_failure("remote lock rpc failed: temporary network issue"));
+        assert!(is_remote_lock_rpc_failure("Remote lock RPC timed out: RPC timed out after 50ms"));
+        assert!(!is_remote_lock_rpc_failure("Lock is already held"));
+        assert!(!is_remote_lock_rpc_failure("acquired lock failed for other reason"));
+    }
+
+    #[tokio::test]
+    async fn acquire_guard_returns_quorum_error_when_rpc_failures_make_quorum_impossible() {
+        let clients: Vec<Arc<dyn LockClient>> = vec![
+            ResponseClient::new(LockResponse::failure("Remote lock RPC failed: node unavailable", Duration::ZERO)).into_client(),
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO))
+                .with_delay(Duration::from_millis(50))
+                .into_client(),
+            ResponseClient::new(LockResponse::failure("Remote lock RPC failed: connection refused", Duration::ZERO))
+                .into_client(),
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO))
+                .with_delay(Duration::from_millis(50))
+                .into_client(),
+        ];
+        let lock = DistributedLock::new("test".to_string(), clients, 3);
+        let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner")
+            .with_acquire_timeout(Duration::from_secs(1));
+
+        let result = lock.acquire_guard(&request).await;
+
+        assert!(
+            matches!(
+                result,
+                Err(LockError::QuorumNotReached {
+                    required: 3,
+                    achieved: 0
+                })
+            ),
+            "unexpected result: {result:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn acquire_guard_returns_quorum_error_when_rpc_timeouts_make_quorum_impossible() {
+        let clients: Vec<Arc<dyn LockClient>> = vec![
+            ResponseClient::new(LockResponse::failure(
+                "Remote lock RPC timed out: RPC timed out after 50ms",
+                Duration::ZERO,
+            ))
+            .into_client(),
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO))
+                .with_delay(Duration::from_millis(50))
+                .into_client(),
+            ResponseClient::new(LockResponse::failure(
+                "Remote lock RPC timed out: RPC timed out after 50ms",
+                Duration::ZERO,
+            ))
+            .into_client(),
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO))
+                .with_delay(Duration::from_millis(50))
+                .into_client(),
+        ];
+        let lock = DistributedLock::new("test".to_string(), clients, 3);
+        let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner")
+            .with_acquire_timeout(Duration::from_secs(1));
+
+        let result = lock.acquire_guard(&request).await;
+
+        assert!(
+            matches!(
+                result,
+                Err(LockError::QuorumNotReached {
+                    required: 3,
+                    achieved: 0
+                })
+            ),
+            "unexpected result: {result:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn acquire_guard_returns_timeout_when_zero_locks_make_quorum_impossible_for_attempt() {
+        let clients: Vec<Arc<dyn LockClient>> = vec![
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO)).into_client(),
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO)).into_client(),
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO))
+                .with_delay(Duration::from_secs(1))
+                .into_client(),
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO))
+                .with_delay(Duration::from_secs(1))
+                .into_client(),
+        ];
+        let lock = DistributedLock::new("test".to_string(), clients, 3);
+        let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner")
+            .with_acquire_timeout(Duration::from_secs(2));
+
+        let started = tokio::time::Instant::now();
+        let result = lock.acquire_guard(&request).await;
+
+        assert!(matches!(result, Ok(None)), "unexpected result: {result:?}");
+        assert!(
+            started.elapsed() < Duration::from_secs(1),
+            "acquire should fail this attempt before waiting for delayed impossible-quorum tasks"
+        );
+    }
+
+    #[tokio::test]
+    async fn acquire_guard_retries_transient_timeout_before_quorum() {
+        let clients: Vec<Arc<dyn LockClient>> = vec![
+            ResponseClient::new(LockResponse::failure(
+                "Timeout { resource: \"bucket/object-flaky-acquire@latest\", timeout: 1s }",
+                Duration::ZERO,
+            ))
+            .into_client(),
+            ResponseClient::new(LockResponse::failure(
+                "Timeout { resource: \"bucket/object-flaky-acquire@latest\", timeout: 1s }",
+                Duration::ZERO,
+            ))
+            .into_client(),
+            ResponseClient::new(LockResponse::failure(
+                "Timeout { resource: \"bucket/object-flaky-acquire@latest\", timeout: 1s }",
+                Duration::ZERO,
+            ))
+            .into_client(),
+            ResponseClient::new(LockResponse::failure(
+                "Timeout { resource: \"bucket/object-flaky-acquire@latest\", timeout: 1s }",
+                Duration::ZERO,
+            ))
+            .into_client(),
+        ];
+        let lock = DistributedLock::new("test".to_string(), clients, 3);
+        let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner")
+            .with_acquire_timeout(Duration::from_millis(900));
+
+        let started = tokio::time::Instant::now();
+        let result = lock.acquire_guard(&request).await;
+        let elapsed = started.elapsed();
+
+        assert!(matches!(result, Ok(None)), "unexpected result: {result:?}");
+        assert!(
+            elapsed >= Duration::from_millis(250),
+            "expected at least one retry attempt for transient timeout, got {elapsed:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn acquire_guard_returns_timeout_when_quorum_remains_contended() {
+        let clients: Vec<Arc<dyn LockClient>> = vec![
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO)).into_client(),
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO)).into_client(),
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO)).into_client(),
+            ResponseClient::new(LockResponse::failure("lock already held", Duration::ZERO)).into_client(),
+        ];
+        let lock = DistributedLock::new("test".to_string(), clients, 3);
+        let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner")
+            .with_acquire_timeout(Duration::from_millis(120));
+
+        let result = lock.acquire_guard(&request).await;
+
+        assert!(matches!(result, Ok(None)), "unexpected result: {result:?}");
+    }
+
+    #[tokio::test]
+    async fn acquire_guard_uses_fresh_lock_ids_across_retry_attempts() {
+        let seen_ids = Arc::new(Mutex::new(Vec::new()));
+        let delayed_success_a = Arc::new(SequencedClient::new(
+            vec![
+                AcquirePlan::Success {
+                    delay: Duration::from_millis(400),
+                },
+                AcquirePlan::Success { delay: Duration::ZERO },
+            ],
+            seen_ids.clone(),
+        ));
+        let delayed_success_b = Arc::new(SequencedClient::new(
+            vec![
+                AcquirePlan::Success {
+                    delay: Duration::from_millis(400),
+                },
+                AcquirePlan::Success { delay: Duration::ZERO },
+            ],
+            seen_ids.clone(),
+        ));
+
+        let clients: Vec<Arc<dyn LockClient>> = vec![
+            Arc::new(SequencedClient::new(
+                vec![
+                    AcquirePlan::Failure {
+                        error: "Lock acquisition timeout",
+                        delay: Duration::ZERO,
+                    },
+                    AcquirePlan::Success { delay: Duration::ZERO },
+                ],
+                seen_ids.clone(),
+            )),
+            Arc::new(SequencedClient::new(
+                vec![
+                    AcquirePlan::Failure {
+                        error: "Lock acquisition timeout",
+                        delay: Duration::ZERO,
+                    },
+                    AcquirePlan::Success { delay: Duration::ZERO },
+                ],
+                seen_ids.clone(),
+            )),
+            delayed_success_a.clone(),
+            delayed_success_b.clone(),
+        ];
+
+        let lock = DistributedLock::new("test".to_string(), clients, 3);
+        let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner")
+            .with_acquire_timeout(Duration::from_secs(2));
+
+        let guard = lock
+            .acquire_guard(&request)
+            .await
+            .expect("retry path should not fail")
+            .expect("second attempt should reach quorum");
+
+        tokio::time::sleep(Duration::from_millis(700)).await;
+
+        let mut unique_ids = Vec::new();
+        for lock_id in seen_ids.lock().unwrap().iter() {
+            if unique_ids.iter().all(|id: &LockId| id.uuid != lock_id.uuid) {
+                unique_ids.push(lock_id.clone());
+            }
+        }
+
+        assert!(
+            unique_ids.len() >= 2,
+            "expected retry attempts to use distinct lock ids, saw {unique_ids:?}"
+        );
+
+        let retry_lock_ids = unique_ids.iter().skip(1).cloned().collect::<Vec<_>>();
+        assert!(
+            !retry_lock_ids.is_empty(),
+            "expected at least one retry-attempt lock id, saw {unique_ids:?}"
+        );
+
+        let delayed_a_active = delayed_success_a.active.lock().await;
+        let delayed_b_active = delayed_success_b.active.lock().await;
+        let remaining_delayed_lock_ids = delayed_a_active
+            .keys()
+            .chain(delayed_b_active.keys())
+            .cloned()
+            .collect::<Vec<_>>();
+
+        assert!(
+            remaining_delayed_lock_ids.len() == 1,
+            "exactly one delayed client lock should remain held by the retry guard after late cleanup"
+        );
+        assert!(
+            retry_lock_ids
+                .iter()
+                .any(|retry_lock_id| retry_lock_id.uuid == remaining_delayed_lock_ids[0].uuid),
+            "late cleanup must not leave only a first-attempt delayed lock active"
+        );
+        drop(delayed_b_active);
+        drop(delayed_a_active);
+
+        drop(guard);
+    }
+}
@@ -64,6 +64,50 @@ impl crate::client::LockClient for FailingClient {
    }
 }

+#[derive(Debug)]
+struct FailureResponseClient {
+    error: &'static str,
+}
+
+#[async_trait::async_trait]
+impl crate::client::LockClient for FailureResponseClient {
+    async fn acquire_lock(&self, _request: &LockRequest) -> crate::Result<LockResponse> {
+        Ok(LockResponse::failure(self.error, Duration::ZERO))
+    }
+
+    async fn release(&self, _lock_id: &LockId) -> crate::Result<bool> {
+        Ok(false)
+    }
+
+    async fn refresh(&self, _lock_id: &LockId) -> crate::Result<bool> {
+        Ok(false)
+    }
+
+    async fn force_release(&self, _lock_id: &LockId) -> crate::Result<bool> {
+        Ok(false)
+    }
+
+    async fn check_status(&self, _lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
+        Ok(None)
+    }
+
+    async fn get_stats(&self) -> crate::Result<LockStats> {
+        Ok(LockStats::default())
+    }
+
+    async fn close(&self) -> crate::Result<()> {
+        Ok(())
+    }
+
+    async fn is_online(&self) -> bool {
+        true
+    }
+
+    async fn is_local(&self) -> bool {
+        false
+    }
+}
+
 #[derive(Debug)]
 struct DelayedClient {
    inner: Arc<dyn crate::client::LockClient>,
@@ -110,6 +154,75 @@ impl crate::client::LockClient for DelayedClient {
    }
 }

+#[derive(Debug)]
+struct FlakyAcquireClient {
+    inner: LocalClient,
+    failed_acquires_remaining: AtomicUsize,
+    acquire_attempts: AtomicUsize,
+}
+
+impl FlakyAcquireClient {
+    fn new(manager: Arc<GlobalLockManager>, failed_acquires: usize) -> Self {
+        Self {
+            inner: LocalClient::with_manager(manager),
+            failed_acquires_remaining: AtomicUsize::new(failed_acquires),
+            acquire_attempts: AtomicUsize::new(0),
+        }
+    }
+
+    fn acquire_attempts(&self) -> usize {
+        self.acquire_attempts.load(Ordering::SeqCst)
+    }
+}
+
+#[async_trait::async_trait]
+impl crate::client::LockClient for FlakyAcquireClient {
+    async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
+        self.acquire_attempts.fetch_add(1, Ordering::SeqCst);
+        if self
+            .failed_acquires_remaining
+            .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
+            .is_ok()
+        {
+            return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
+        }
+
+        self.inner.acquire_lock(request).await
+    }
+
+    async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.inner.release(lock_id).await
+    }
+
+    async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.inner.refresh(lock_id).await
+    }
+
+    async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.inner.force_release(lock_id).await
+    }
+
+    async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
+        self.inner.check_status(lock_id).await
+    }
+
+    async fn get_stats(&self) -> crate::Result<LockStats> {
+        self.inner.get_stats().await
+    }
+
+    async fn close(&self) -> crate::Result<()> {
+        self.inner.close().await
+    }
+
+    async fn is_online(&self) -> bool {
+        self.inner.is_online().await
+    }
+
+    async fn is_local(&self) -> bool {
+        self.inner.is_local().await
+    }
+}
+
 #[derive(Debug)]
 struct FlakyReleaseClient {
    inner: LocalClient,
@@ -633,10 +746,10 @@ async fn test_namespace_lock_distributed_eight_node_write_releases_all_nodes() {
        .get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
        .await
        .expect_err("owner-b should not acquire while owner-a holds all node locks");
-    let err_str = err.to_string();
+    let err_str = err.to_string().to_lowercase();
    assert!(
-        err_str.contains("required 5") && err_str.contains("achieved"),
-        "expected 8-node quorum failure below required write quorum, got: {err}"
+        err_str.contains("timeout"),
+        "expected owner-b contention to exhaust acquire timeout, got: {err}"
    );

    assert!(guard.release(), "distributed guard should enqueue release");
@@ -672,6 +785,69 @@ async fn test_namespace_lock_distributed_unlock_retries_release_false() {
    );
 }

+#[tokio::test]
+async fn test_namespace_lock_distributed_retries_transient_acquire_timeout() {
+    let managers = (0..3).map(|_| Arc::new(GlobalLockManager::new())).collect::<Vec<_>>();
+    let flaky_clients = managers
+        .iter()
+        .map(|manager| Arc::new(FlakyAcquireClient::new(manager.clone(), 1)))
+        .collect::<Vec<_>>();
+    let clients = flaky_clients
+        .iter()
+        .map(|client| client.clone() as Arc<dyn LockClient>)
+        .collect::<Vec<_>>();
+
+    let lock = NamespaceLock::with_clients("flaky-acquire".to_string(), clients);
+    let resource = create_test_object_key("bucket", "object-flaky-acquire");
+
+    let guard = lock
+        .get_write_lock(resource, "owner-a", Duration::from_secs(1))
+        .await
+        .expect("transient timeout should be retried before the acquire budget expires");
+
+    assert!(
+        flaky_clients.iter().all(|client| client.acquire_attempts() >= 2),
+        "each simulated node should be retried after an initial timeout"
+    );
+
+    drop(guard);
+}
+
+#[tokio::test]
+async fn test_namespace_lock_distributed_waits_full_timeout_for_late_release() {
+    let managers = (0..3).map(|_| Arc::new(GlobalLockManager::new())).collect::<Vec<_>>();
+    let clients = managers
+        .iter()
+        .map(|manager| Arc::new(LocalClient::with_manager(manager.clone())) as Arc<dyn LockClient>)
+        .collect::<Vec<_>>();
+
+    let lock = Arc::new(NamespaceLock::with_clients("late-release".to_string(), clients));
+    let resource = create_test_object_key("bucket", "object-late-release");
+
+    let guard_a = lock
+        .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(1))
+        .await
+        .expect("owner-a should acquire the initial distributed lock");
+
+    let lock_for_owner_b = lock.clone();
+    let resource_for_owner_b = resource.clone();
+    let waiter = tokio::spawn(async move {
+        lock_for_owner_b
+            .get_write_lock(resource_for_owner_b, "owner-b", Duration::from_secs(1))
+            .await
+    });
+
+    tokio::time::sleep(Duration::from_millis(950)).await;
+    drop(guard_a);
+
+    let guard_b = waiter
+        .await
+        .expect("owner-b wait task should complete")
+        .expect("owner-b should acquire after a late release within the original timeout");
+
+    drop(guard_b);
+}
+
 #[test]
 fn test_namespace_lock_distributed_drop_without_runtime_does_not_panic() {
    let (manager, resource, guard) = {
@@ -745,6 +921,68 @@ async fn test_namespace_lock_distributed_write_lock_fails_with_two_nodes_one_off
    );
 }

+#[tokio::test]
+async fn test_namespace_lock_distributed_remote_rpc_failures_are_hard_quorum_failures() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let client_ok: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager));
+    let client_rpc_failed: Arc<dyn LockClient> = Arc::new(FailureResponseClient {
+        error: "Remote lock RPC failed: connection refused",
+    });
+    let client_rpc_timed_out: Arc<dyn LockClient> = Arc::new(FailureResponseClient {
+        error: "Remote lock RPC timed out: RPC timed out after 50ms",
+    });
+    let client_rpc_failed_2: Arc<dyn LockClient> = Arc::new(FailureResponseClient {
+        error: "Remote lock RPC failed: transport error",
+    });
+    let clients: Vec<Arc<dyn LockClient>> = vec![client_ok, client_rpc_failed, client_rpc_timed_out, client_rpc_failed_2];
+    let lock = NamespaceLock::with_clients("remote-rpc-hard-failure".to_string(), clients);
+    let resource = create_test_object_key("bucket", "object-rpc-hard-failure");
+
+    let started = tokio::time::Instant::now();
+    let err = lock
+        .get_write_lock(resource, "owner-a", Duration::from_secs(1))
+        .await
+        .expect_err("write lock should fail as soon as remote RPC failures make quorum impossible");
+
+    assert!(
+        started.elapsed() < Duration::from_millis(150),
+        "remote RPC failures should not retry until the full acquire timeout"
+    );
+    let err_str = err.to_string().to_lowercase();
+    assert!(
+        err_str.contains("quorum") || err_str.contains("not reached"),
+        "expected hard quorum failure, got: {err}"
+    );
+}
+
+#[tokio::test]
+async fn test_namespace_lock_distributed_contention_quorum_miss_times_out() {
+    let client_timeout_1: Arc<dyn LockClient> = Arc::new(FailureResponseClient {
+        error: "Lock acquisition timeout",
+    });
+    let client_timeout_2: Arc<dyn LockClient> = Arc::new(FailureResponseClient {
+        error: "Lock acquisition timeout",
+    });
+    let client_timeout_3: Arc<dyn LockClient> = Arc::new(FailureResponseClient {
+        error: "Lock acquisition timeout",
+    });
+    let clients: Vec<Arc<dyn LockClient>> = vec![client_timeout_1, client_timeout_2, client_timeout_3];
+    let lock = NamespaceLock::with_clients("contention-timeout".to_string(), clients);
+    let resource = create_test_object_key("bucket", "object-contention-timeout");
+
+    let err = lock
+        .get_write_lock(resource, "owner-a", Duration::from_millis(40))
+        .await
+        .expect_err("ordinary contention should exhaust acquire timeout");
+
+    let err_str = err.to_string().to_lowercase();
+    assert!(err_str.contains("timeout"), "expected timeout after contention retries, got: {err}");
+    assert!(
+        !err_str.contains("quorum not reached"),
+        "ordinary contention should not surface as quorum loss: {err}"
+    );
+}
+
 #[tokio::test]
 async fn test_namespace_lock_distributed_quorum_failure_rolls_back_successful_nodes() {
    let manager1 = Arc::new(GlobalLockManager::new());
@@ -976,7 +976,7 @@ pub async fn start_background_task(disks: Vec<CapacityDiskRef>) {
    }

    tokio::spawn(async move {
-        let mut timer = tokio::time::interval(refresh_interval);
+        let mut timer = tokio::time::interval_at(tokio::time::Instant::now() + refresh_interval, refresh_interval);

        loop {
            timer.tick().await;
@@ -1002,7 +1002,7 @@ pub async fn start_background_task(disks: Vec<CapacityDiskRef>) {
    });

    tokio::spawn(async move {
-        let mut timer = tokio::time::interval(metrics_interval);
+        let mut timer = tokio::time::interval_at(tokio::time::Instant::now() + metrics_interval, metrics_interval);
        loop {
            timer.tick().await;
            manager_for_metrics.log_runtime_summary().await;
@@ -192,7 +192,7 @@ impl Resource {
                return true;
            }

-            if wildcard::is_match(resolved_pattern, resource) {
+            if cp != "." && wildcard::is_match(resolved_pattern, &cp) {
                return true;
            }
        }
@@ -287,6 +287,8 @@ mod tests {
    #[test_case("arn:aws:s3:::mybucket/*","mybucket10/myobject" => false; "13")]
    #[test_case("arn:aws:s3:::mybucket?0/2010/photos/*","mybucket0/2010/photos/1.jpg" => false; "14")]
    #[test_case("arn:aws:s3:::mybucket","mybucket/myobject" => false; "15")]
+    #[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/../victim-bucket/evil.txt" => false; "16")]
+    #[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/safe/../../victim-bucket/evil.txt" => false; "17")]
    fn test_resource_is_match(resource: &str, object: &str) -> bool {
        let resource: Resource = resource.try_into().unwrap();

@@ -192,6 +192,10 @@ where
        let mut client_builder = Client::builder()
            .timeout(Duration::from_secs(30))
            .user_agent(crate::get_user_agent(crate::ServiceType::Basis));
+        #[cfg(test)]
+        {
+            client_builder = client_builder.no_proxy();
+        }

        // 1. Configure server certificate verification
        if args.skip_tls_verify {
@@ -560,7 +564,6 @@ mod tests {
    use super::{WebhookArgs, WebhookTarget};
    use crate::target::{Target, TargetType, decode_object_name};
    use tokio::net::TcpListener;
-    use tokio::sync::mpsc;
    use url::Url;
    use url::form_urlencoded;

@@ -679,43 +682,37 @@ mod tests {
    async fn test_is_active_uses_origin_reachability_for_path_endpoints() {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();
-        let (path_tx, mut path_rx) = mpsc::channel(1);
-        let accept_task = tokio::spawn(async move {
+        let server = async move {
+            use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+            let (mut stream, _) = listener.accept().await.unwrap();
+            let mut request = Vec::new();
+            let mut buf = [0u8; 1024];
            loop {
-                let (mut stream, _) = listener.accept().await.unwrap();
-                let path_tx = path_tx.clone();
-                tokio::spawn(async move {
-                    use tokio::io::{AsyncReadExt, AsyncWriteExt};
-
-                    let mut request = Vec::new();
-                    let mut buf = [0u8; 1024];
-                    loop {
-                        let read = stream.read(&mut buf).await.unwrap();
-                        if read == 0 {
-                            break;
-                        }
-                        request.extend_from_slice(&buf[..read]);
-                        if request.windows(4).any(|window| window == b"\r\n\r\n") {
-                            break;
-                        }
-                    }
-
-                    let request_line = request
-                        .split(|byte| *byte == b'\n')
-                        .next()
-                        .and_then(|line| std::str::from_utf8(line).ok())
-                        .unwrap_or_default()
-                        .trim();
-                    let path = request_line.split_whitespace().nth(1).unwrap_or_default().to_string();
-                    let _ = path_tx.send(path.clone()).await;
-
-                    if path == "/" {
-                        let response = b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";
-                        let _ = stream.write_all(response).await;
-                    }
-                });
+                let read = stream.read(&mut buf).await.unwrap();
+                if read == 0 {
+                    break;
+                }
+                request.extend_from_slice(&buf[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
            }
-        });
+
+            let request_line = request
+                .split(|byte| *byte == b'\n')
+                .next()
+                .and_then(|line| std::str::from_utf8(line).ok())
+                .unwrap_or_default()
+                .trim();
+            let path = request_line.split_whitespace().nth(1).unwrap_or_default().to_string();
+
+            if path == "/" {
+                let response = b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";
+                let _ = stream.write_all(response).await;
+            }
+            path
+        };

        let args = WebhookArgs {
            endpoint: Url::parse(&format!("http://{address}/hook")).unwrap(),
@@ -723,8 +720,8 @@ mod tests {
        };
        let target = WebhookTarget::<serde_json::Value>::new("path-probe".to_string(), args).unwrap();

-        assert!(target.is_active().await.unwrap());
-        assert_eq!(path_rx.recv().await.unwrap(), "/");
-        accept_task.abort();
+        let (is_active, path) = tokio::join!(target.is_active(), server);
+        assert!(is_active.unwrap());
+        assert_eq!(path, "/");
    }
 }
@@ -20,7 +20,7 @@ use rustls::sign::CertifiedKey;
 use rustls_pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject};
 use std::collections::HashMap;
 use std::io::Error;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::{fs, io};
 use tracing::{debug, warn};
@@ -224,10 +224,118 @@ pub fn certs_error(err: String) -> Error {
    Error::other(err)
 }

+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum TlsCertPairStatus {
+    MissingBoth,
+    MissingCert,
+    MissingKey,
+    Valid,
+    Invalid { error: String },
+}
+
+impl TlsCertPairStatus {
+    pub fn is_valid(&self) -> bool {
+        matches!(self, Self::Valid)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TlsCertPairInspection {
+    pub cert_path: PathBuf,
+    pub key_path: PathBuf,
+    pub status: TlsCertPairStatus,
+}
+
+impl TlsCertPairInspection {
+    pub fn is_valid(&self) -> bool {
+        self.status.is_valid()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TlsDomainInspection {
+    pub domain_name: String,
+    pub pair: TlsCertPairInspection,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TlsDirectoryInspection {
+    pub directory: PathBuf,
+    pub canonical_directory: Option<PathBuf>,
+    pub root_pair: TlsCertPairInspection,
+    pub domain_pairs: Vec<TlsDomainInspection>,
+    pub skipped_directory_names: Vec<String>,
+}
+
+impl TlsDirectoryInspection {
+    pub fn valid_domain_names(&self) -> Vec<&str> {
+        self.domain_pairs
+            .iter()
+            .filter(|entry| entry.pair.is_valid())
+            .map(|entry| entry.domain_name.as_str())
+            .collect()
+    }
+
+    pub fn has_valid_root_pair(&self) -> bool {
+        self.root_pair.is_valid()
+    }
+}
+
 fn is_discoverable_cert_domain_dir(domain_name: &str) -> bool {
    !domain_name.starts_with('.')
 }

+pub fn inspect_cert_directory(options: CertDirectoryLoadOptions) -> io::Result<TlsDirectoryInspection> {
+    options.validate()?;
+
+    let dir = options.dir_path.as_path();
+    if !dir.exists() || !dir.is_dir() {
+        return Err(certs_error(format!(
+            "The certificate directory does not exist or is not a directory: {}",
+            dir.display()
+        )));
+    }
+
+    let root_pair = inspect_cert_key_pair(dir, &options.cert_filename, &options.key_filename);
+    let canonical_directory = fs::canonicalize(dir).ok();
+    let mut domain_pairs = Vec::new();
+    let mut skipped_directory_names = Vec::new();
+
+    for entry in fs::read_dir(dir)? {
+        let entry = entry?;
+        let path = entry.path();
+
+        if !path.is_dir() {
+            continue;
+        }
+
+        let domain_name = path
+            .file_name()
+            .and_then(|name| name.to_str())
+            .ok_or_else(|| certs_error(format!("invalid domain name directory:{path:?}")))?;
+        if !is_discoverable_cert_domain_dir(domain_name) {
+            skipped_directory_names.push(domain_name.to_string());
+            continue;
+        }
+
+        domain_pairs.push(TlsDomainInspection {
+            domain_name: domain_name.to_string(),
+            pair: inspect_cert_key_pair(&path, &options.cert_filename, &options.key_filename),
+        });
+    }
+
+    domain_pairs.sort_by(|left, right| left.domain_name.cmp(&right.domain_name));
+    skipped_directory_names.sort();
+
+    Ok(TlsDirectoryInspection {
+        directory: dir.to_path_buf(),
+        canonical_directory,
+        root_pair,
+        domain_pairs,
+        skipped_directory_names,
+    })
+}
+
 pub fn load_all_certs_from_directory(
    options: CertDirectoryLoadOptions,
 ) -> io::Result<HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>> {
@@ -327,6 +435,42 @@ fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec<Certif
    Ok((certs, key))
 }

+fn inspect_cert_key_pair(dir: &Path, cert_filename: &str, key_filename: &str) -> TlsCertPairInspection {
+    let cert_path = dir.join(cert_filename);
+    let key_path = dir.join(key_filename);
+    let cert_exists = cert_path.exists();
+    let key_exists = key_path.exists();
+
+    let status = match (cert_exists, key_exists) {
+        (false, false) => TlsCertPairStatus::MissingBoth,
+        (false, true) => TlsCertPairStatus::MissingCert,
+        (true, false) => TlsCertPairStatus::MissingKey,
+        (true, true) => match cert_key_pair_utf8_paths(&cert_path, &key_path) {
+            Ok((cert_path, key_path)) => match load_cert_key_pair(cert_path, key_path) {
+                Ok(_) => TlsCertPairStatus::Valid,
+                Err(err) => TlsCertPairStatus::Invalid { error: err.to_string() },
+            },
+            Err(err) => TlsCertPairStatus::Invalid { error: err.to_string() },
+        },
+    };
+
+    TlsCertPairInspection {
+        cert_path,
+        key_path,
+        status,
+    }
+}
+
+fn cert_key_pair_utf8_paths<'a>(cert_path: &'a Path, key_path: &'a Path) -> io::Result<(&'a str, &'a str)> {
+    let cert_path = cert_path
+        .to_str()
+        .ok_or_else(|| certs_error(format!("Invalid UTF-8 in certificate path: {cert_path:?}")))?;
+    let key_path = key_path
+        .to_str()
+        .ok_or_else(|| certs_error(format!("Invalid UTF-8 in key path: {key_path:?}")))?;
+    Ok((cert_path, key_path))
+}
+
 pub fn create_multi_cert_resolver(
    cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
 ) -> io::Result<impl ResolvesServerCert> {
@@ -448,4 +592,36 @@ mod tests {
        assert!(!certs.contains_key("..data"));
        assert_eq!(certs.len(), 2);
    }
+
+    #[test]
+    fn test_inspect_cert_directory_reports_valid_root_and_domain_pairs() {
+        let temp_dir = TempDir::new().expect("tempdir should create");
+        write_test_cert_pair(temp_dir.path());
+
+        let domain_dir = temp_dir.path().join("example.com");
+        fs::create_dir(&domain_dir).expect("domain dir should create");
+        write_test_cert_pair(&domain_dir);
+
+        let inspection = inspect_cert_directory(default_load_options(temp_dir.path())).expect("inspection should succeed");
+        assert!(inspection.has_valid_root_pair());
+        assert_eq!(inspection.valid_domain_names(), vec!["example.com"]);
+        assert_eq!(inspection.domain_pairs.len(), 1);
+        assert!(inspection.domain_pairs[0].pair.is_valid());
+    }
+
+    #[test]
+    fn test_inspect_cert_directory_reports_invalid_and_missing_pairs() {
+        let temp_dir = TempDir::new().expect("tempdir should create");
+        fs::write(temp_dir.path().join("rustfs_cert.pem"), "invalid certificate").expect("invalid cert should write");
+        fs::write(temp_dir.path().join("rustfs_key.pem"), "invalid key").expect("invalid key should write");
+
+        let domain_dir = temp_dir.path().join("broken.example.com");
+        fs::create_dir(&domain_dir).expect("domain dir should create");
+        fs::write(domain_dir.join("rustfs_cert.pem"), "invalid certificate").expect("invalid cert should write");
+
+        let inspection = inspect_cert_directory(default_load_options(temp_dir.path())).expect("inspection should succeed");
+        assert!(matches!(inspection.root_pair.status, TlsCertPairStatus::Invalid { .. }));
+        assert_eq!(inspection.domain_pairs.len(), 1);
+        assert_eq!(inspection.domain_pairs[0].pair.status, TlsCertPairStatus::MissingKey);
+    }
 }
@@ -26,7 +26,8 @@ pub mod source;
 pub mod state;

 pub use certs::{
-    CertDirectoryLoadOptions, WebPkiClientVerifierOptions, build_webpki_client_verifier, create_multi_cert_resolver,
+    CertDirectoryLoadOptions, TlsCertPairInspection, TlsCertPairStatus, TlsDirectoryInspection, TlsDomainInspection,
+    WebPkiClientVerifierOptions, build_webpki_client_verifier, create_multi_cert_resolver, inspect_cert_directory,
    load_all_certs_from_directory, load_cert_bundle_der_bytes, load_certs, load_private_key,
 };
 pub use config::{ReloadApplyHint, ReloadDetectMode, TlsReloadOptions};
@@ -19,10 +19,13 @@ use std::fs;
 use std::fs::File;
 use std::io::{self, BufRead, Error, ErrorKind, Read};
 use std::path::{Path, PathBuf};
+use std::sync::{Mutex, OnceLock};
+use tracing::warn;
+
+static BAVAIL_GT_BFREE_WARNING_PATHS: OnceLock<Mutex<BTreeSet<PathBuf>>> = OnceLock::new();

 /// Returns total and free bytes available in a directory, e.g. `/`.
 pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
-    let path_display = p.as_ref().display();
    // Use statfs on Linux to get access to f_type (filesystem magic number)
    let stat = statfs(p.as_ref())?;

@@ -42,34 +45,7 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
    let bfree = stat.f_bfree as u64;
    let bavail = stat.f_bavail as u64;
    let blocks = stat.f_blocks as u64;
-
-    let reserved = match bfree.checked_sub(bavail) {
-        Some(reserved) => reserved,
-        None => {
-            return Err(Error::other(format!(
-                "detected f_bavail space ({bavail}) > f_bfree space ({bfree}), fs corruption at ({path_display}). please run 'fsck'",
-            )));
-        }
-    };
-
-    let total = match blocks.checked_sub(reserved) {
-        Some(total) => total * bsize,
-        None => {
-            return Err(Error::other(format!(
-                "detected reserved space ({reserved}) > blocks space ({blocks}), fs corruption at ({path_display}). please run 'fsck'",
-            )));
-        }
-    };
-
-    let free = bavail * bsize;
-    let used = match total.checked_sub(free) {
-        Some(used) => used,
-        None => {
-            return Err(Error::other(format!(
-                "detected free space ({free}) > total drive space ({total}), fs corruption at ({path_display}). please run 'fsck'"
-            )));
-        }
-    };
+    let (total, free, used) = calculate_space_usage(blocks, bfree, bavail, bsize, p.as_ref())?;

    let st = rustix::fs::stat(p.as_ref())?;

@@ -86,6 +62,59 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
    })
 }

+fn calculate_space_usage(blocks: u64, bfree: u64, bavail: u64, bsize: u64, path: &Path) -> std::io::Result<(u64, u64, u64)> {
+    let available = if bfree < bavail {
+        if should_warn_bavail_greater_than_bfree(path) {
+            warn!(
+                path = %path.display(),
+                f_bfree = bfree,
+                f_bavail = bavail,
+                "detected f_bavail greater than f_bfree, capping available blocks to f_bfree for compatibility"
+            );
+        }
+        bfree
+    } else {
+        bavail
+    };
+    let reserved = bfree - available;
+
+    let total = match blocks.checked_sub(reserved) {
+        Some(total) => total * bsize,
+        None => {
+            return Err(Error::other(format!(
+                "detected reserved space ({reserved}) > blocks space ({blocks}), fs corruption at ({}). please run 'fsck'",
+                path.display(),
+            )));
+        }
+    };
+
+    let free = available * bsize;
+    let used = match total.checked_sub(free) {
+        Some(used) => used,
+        None => {
+            return Err(Error::other(format!(
+                "detected free space ({free}) > total drive space ({total}), fs corruption at ({}). please run 'fsck'",
+                path.display(),
+            )));
+        }
+    };
+
+    Ok((total, free, used))
+}
+
+fn should_warn_bavail_greater_than_bfree(path: &Path) -> bool {
+    let warned_paths = BAVAIL_GT_BFREE_WARNING_PATHS.get_or_init(|| Mutex::new(BTreeSet::new()));
+    let mut warned_paths = match warned_paths.lock() {
+        Ok(guard) => guard,
+        Err(poisoned) => poisoned.into_inner(),
+    };
+    if warned_paths.contains(path) {
+        false
+    } else {
+        warned_paths.insert(path.to_path_buf())
+    }
+}
+
 pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result<bool> {
    let stat1 = rustix::fs::stat(disk1)?;
    let stat2 = rustix::fs::stat(disk2)?;
@@ -407,4 +436,82 @@ mod tests {
        let ids = resolve_block_device_ids(major, minor).unwrap();
        assert_eq!(ids.into_iter().collect::<Vec<_>>(), vec![format!("{major}:{minor}")]);
    }
+
+    #[test]
+    fn calculate_space_usage_normal_bfree_greater_than_bavail() {
+        // Typical ext4/xfs scenario: bavail < bfree due to reserved blocks
+        let blocks = 1_000_u64;
+        let bfree = 900_u64;
+        let bavail = 850_u64;
+        let bsize = 4_096_u64;
+
+        let (total, free, used) = calculate_space_usage(blocks, bfree, bavail, bsize, Path::new("/data")).unwrap();
+        let reserved = bfree - bavail;
+        assert_eq!(total, (blocks - reserved) * bsize);
+        assert_eq!(free, bavail * bsize);
+        assert_eq!(used, total - free);
+    }
+
+    #[test]
+    fn calculate_space_usage_bfree_equals_bavail() {
+        // No reserved blocks: bfree == bavail (e.g. FAT/exFAT or root user)
+        let blocks = 500_u64;
+        let bfree = 400_u64;
+        let bavail = 400_u64;
+        let bsize = 4_096_u64;
+
+        let (total, free, used) = calculate_space_usage(blocks, bfree, bavail, bsize, Path::new("/data")).unwrap();
+        assert_eq!(total, blocks * bsize);
+        assert_eq!(free, bfree * bsize);
+        assert_eq!(used, (blocks - bfree) * bsize);
+    }
+
+    #[test]
+    fn calculate_space_usage_all_zero_blocks() {
+        let (total, free, used) = calculate_space_usage(0, 0, 0, 4_096, Path::new("/data")).unwrap();
+        assert_eq!(total, 0);
+        assert_eq!(free, 0);
+        assert_eq!(used, 0);
+    }
+
+    #[test]
+    fn calculate_space_usage_all_blocks_free() {
+        let blocks = 1_000_u64;
+        let bfree = 1_000_u64;
+        let bavail = 1_000_u64;
+        let bsize = 4_096_u64;
+
+        let (total, free, used) = calculate_space_usage(blocks, bfree, bavail, bsize, Path::new("/data")).unwrap();
+        assert_eq!(total, blocks * bsize);
+        assert_eq!(free, blocks * bsize);
+        assert_eq!(used, 0);
+    }
+
+    #[test]
+    fn calculate_space_usage_allows_bavail_greater_than_bfree() {
+        let blocks = 1_000_u64;
+        let bfree = 900_u64;
+        let bavail = 920_u64;
+        let bsize = 4_096_u64;
+
+        let (total, free, used) = calculate_space_usage(blocks, bfree, bavail, bsize, Path::new("/data")).unwrap();
+        assert_eq!(total, blocks * bsize);
+        assert_eq!(free, bfree * bsize);
+        assert_eq!(used, total - free);
+    }
+
+    #[test]
+    fn calculate_space_usage_rejects_free_greater_than_total() {
+        let err = calculate_space_usage(100, 120, 120, 4_096, Path::new("/data")).unwrap_err();
+        assert!(err.to_string().contains("detected free space"));
+    }
+
+    #[test]
+    fn bavail_greater_than_bfree_warning_is_once_per_path() {
+        let path = Path::new("/data/rustfs-bavail-warning-once");
+
+        assert!(should_warn_bavail_greater_than_bfree(path));
+        assert!(!should_warn_bavail_greater_than_bfree(path));
+        assert!(should_warn_bavail_greater_than_bfree(Path::new("/data/rustfs-bavail-warning-other")));
+    }
 }
@@ -0,0 +1,112 @@
+# RustFS NixOS Service Guide
+
+This guide explains how to use [rustfs.nixos.service.nix](./rustfs.nixos.service.nix) as a starting point for managing RustFS with `systemd` on NixOS.
+
+## 1. What the example does
+
+The NixOS example is designed for the current RustFS startup model:
+
+- `Type = "notify"` so `systemd` waits for RustFS to publish `READY=1`
+- a dedicated `rustfs` system user instead of running the service as `root`
+- `WorkingDirectory = "/var/lib/rustfs"` instead of using a log directory as the working directory
+- `KillMode = "control-group"` and `SendSIGKILL = true` so stuck shutdowns do not leave the unit hanging forever
+- `StandardOutput = "journal"` and `StandardError = "journal"` so logs stay in `journald`
+- runtime configuration through `RUSTFS_*` environment variables
+- secret file wiring through `RUSTFS_ACCESS_KEY_FILE` and `RUSTFS_SECRET_KEY_FILE`
+
+## 2. Import the example into your NixOS configuration
+
+You can either copy the contents into your own module, or import it directly and provide `rustfsPkg`.
+
+Example:
+
+```nix
+{
+  imports = [
+    ./deploy/build/rustfs.nixos.service.nix
+  ];
+
+  _module.args.rustfsPkg = pkgs.callPackage ./path/to/rustfs/package.nix { };
+}
+```
+
+If you already package RustFS through a flake output or overlay, point `rustfsPkg` at that package instead.
+
+## 3. Adjust the default paths
+
+The example uses these defaults:
+
+- data directory: `/srv/rustfs`
+- state directory: `/var/lib/rustfs`
+- log directory: `/var/log/rustfs`
+
+Before enabling the service, make sure your RustFS volumes and any TLS material match your real deployment layout.
+
+The example currently sets:
+
+```nix
+RUSTFS_VOLUMES = "/srv/rustfs/vol{1...4}";
+RUSTFS_ADDRESS = "0.0.0.0:9000";
+RUSTFS_CONSOLE_ENABLE = "true";
+RUSTFS_CONSOLE_ADDRESS = "0.0.0.0:9001";
+```
+
+Update these values to match your storage topology and exposure policy.
+
+## 4. Wire secrets through runtime files
+
+Do not put credentials directly into `environment` if they would be stored in the Nix store.
+
+RustFS supports file-based secrets, so on NixOS the preferred pattern is:
+
+```nix
+systemd.services.rustfs.environment = {
+  RUSTFS_ACCESS_KEY_FILE = config.age.secrets.rustfs-access.path;
+  RUSTFS_SECRET_KEY_FILE = config.age.secrets.rustfs-secret.path;
+};
+```
+
+The same shape also works with `sops-nix`, `agenix`, or any other runtime secret manager that exposes files under `/run`.
+
+## 5. Rebuild and enable the service
+
+After integrating the unit into your NixOS configuration:
+
+```bash
+sudo nixos-rebuild switch
+sudo systemctl enable --now rustfs
+```
+
+## 6. Verify readiness and logs
+
+Check whether `systemd` sees the service as fully ready:
+
+```bash
+systemctl status rustfs
+systemctl show rustfs -p Type -p ActiveState -p SubState
+```
+
+Follow the service logs:
+
+```bash
+journalctl -u rustfs -f
+```
+
+Look for the point where RustFS reports that startup is complete and `systemd` transitions the unit into the running state.
+
+## 7. Common NixOS-specific notes
+
+- If you previously used `SendSIGKILL = false`, expect shutdown behavior to change. The example favors reliable service recovery over indefinite graceful waits.
+- If your service really needs more startup time, increase `TimeoutStartSec`, but keep it bounded. Very large values can hide broken startup states.
+- If you must write file logs instead of journald logs, change `StandardOutput` and `StandardError`, then make sure the target path is writable by the `rustfs` user.
+- If you decide to run as `root`, review the hardening flags again. The example is written for a dedicated unprivileged service user.
+
+## 8. Validation note
+
+If your deployment depends on this example directly, validate the final module in your own NixOS environment before rollout. A common check is:
+
+```bash
+nix-instantiate --parse /path/to/your/module.nix
+```
+
+If you integrate the example into a host configuration or flake, prefer validating it through your normal `nixos-rebuild`, flake check, or CI pipeline as well.
@@ -0,0 +1,107 @@
+{
+  config,
+  lib,
+  pkgs,
+  rustfsPkg,
+  ...
+}:
+let
+  dataDir = "/srv/rustfs";
+  stateDir = "/var/lib/rustfs";
+  logDir = "/var/log/rustfs";
+in
+{
+  users.groups.rustfs = { };
+
+  users.users.rustfs = {
+    isSystemUser = true;
+    group = "rustfs";
+    home = stateDir;
+    createHome = false;
+    description = "RustFS service user";
+  };
+
+  systemd.services.rustfs = {
+    description = "RustFS Object Storage Server";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+
+    # Keep non-secret runtime settings in the unit environment. For secrets,
+    # prefer runtime files so credentials do not land in the Nix store.
+    environment = {
+      RUSTFS_ADDRESS = "0.0.0.0:9000";
+      RUSTFS_CONSOLE_ENABLE = "true";
+      RUSTFS_CONSOLE_ADDRESS = "0.0.0.0:9001";
+      RUSTFS_VOLUMES = "${dataDir}/vol{1...4}";
+    };
+
+    serviceConfig = {
+      Type = "notify";
+      NotifyAccess = "main";
+
+      User = "rustfs";
+      Group = "rustfs";
+
+      # Use a state directory instead of a log directory as the cwd so
+      # relative paths do not accidentally resolve under /var/log.
+      WorkingDirectory = stateDir;
+      StateDirectory = "rustfs";
+      LogsDirectory = "rustfs";
+
+      # Explicitly use the server entrypoint instead of relying on legacy
+      # CLI compatibility shims in service management.
+      ExecStart = "${rustfsPkg}/bin/rustfs server";
+
+      LimitNOFILE = 1048576;
+      LimitNPROC = 32768;
+      TasksMax = "infinity";
+
+      Restart = "on-failure";
+      RestartSec = "10s";
+      StartLimitIntervalSec = "5min";
+      StartLimitBurst = 5;
+
+      # RustFS publishes READY=1 after runtime readiness succeeds. These
+      # values leave room for cold starts without hiding permanently wedged
+      # instances for too long.
+      TimeoutStartSec = "5min";
+      TimeoutStopSec = "45s";
+      KillMode = "control-group";
+      SendSIGKILL = true;
+
+      # Keep the example neutral by default. If your deployment explicitly
+      # wants RustFS to outlive other workloads during OOM, tune this in your
+      # local NixOS configuration.
+      OOMScoreAdjust = 0;
+
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+      ProtectHome = true;
+      ProtectSystem = "strict";
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      UMask = "0077";
+
+      ReadWritePaths = [
+        dataDir
+        stateDir
+        logDir
+      ];
+
+      StandardOutput = "journal";
+      StandardError = "journal";
+    };
+  };
+
+  # Example secret wiring for sops-nix / agenix style runtime files:
+  #
+  # systemd.services.rustfs.environment = {
+  #   RUSTFS_ACCESS_KEY_FILE = config.age.secrets.rustfs-access.path;
+  #   RUSTFS_SECRET_KEY_FILE = config.age.secrets.rustfs-secret.path;
+  # };
+}
@@ -16,17 +16,11 @@ Group=rustfs
 # working directory
 WorkingDirectory=/opt/rustfs

-# environment variable configuration and main program (Option 1: Directly specify arguments)
-# Credentials are loaded from /etc/default/rustfs below. Replace the sample values before deployment.
-ExecStart=/usr/local/bin/rustfs \
-    --address 0.0.0.0:9000 \
-    --volumes /data/rustfs/vol1,/data/rustfs/vol2 \
-    --console-enable
-
-# environment variable configuration (Option 2: Use environment variables)
-# rustfs example file see: `../config/rustfs.env`
+# Environment-driven startup.
+# rustfs reads address, volumes, console, credentials, and other runtime settings
+# from RUSTFS_* variables. See `../config/rustfs.env` for an example template.
 EnvironmentFile=/etc/default/rustfs
-ExecStart=/usr/local/bin/rustfs  $RUSTFS_VOLUMES
+ExecStart=/usr/local/bin/rustfs server

 # service log configuration
 LogsDirectory=rustfs
@@ -43,8 +37,10 @@ Restart=always
 RestartSec=10s

 # graceful exit configuration
-TimeoutStartSec=30s
-TimeoutStopSec=30s
+# Type=notify waits for READY=1, so startup timeout must cover RustFS initialization
+# plus runtime readiness checks on slower disks or cold starts.
+TimeoutStartSec=120s
+TimeoutStopSec=45s

 # security settings
 NoNewPrivileges=true
@@ -33,8 +33,10 @@ services:
      - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
      - RUSTFS_CONSOLE_ENABLE=true
      - RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS=*
-      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:?Set RUSTFS_ACCESS_KEY to a non-default value}
-      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:?Set RUSTFS_SECRET_KEY to a non-default value}
+      # SECURITY: these defaults are public and well-known.
+      # Override via env vars before exposing the listener beyond localhost.
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
      - RUSTFS_OBS_LOGGER_LEVEL=info
      - RUSTFS_TLS_PATH=/opt/tls
      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
@@ -114,7 +114,7 @@ process_log_directory

 # 4) Default credentials warning
 if [ "${RUSTFS_ACCESS_KEY:-}" = "rustfsadmin" ] || [ "${RUSTFS_SECRET_KEY:-}" = "rustfsadmin" ]; then
-  echo "!!!WARNING: Default credentials are only allowed on loopback or with explicit insecure local-dev opt-in."
+  echo "!!!WARNING: Using default RUSTFS_ACCESS_KEY or RUSTFS_SECRET_KEY. Override them in production!"
 fi

 # 5) Append DATA_VOLUMES only if no data paths in arguments
@@ -60,7 +60,7 @@
        {
          default = rustPlatform.buildRustPackage {
            pname = "rustfs";
-            version = "1.0.0-beta.4";
+            version = "1.0.0-beta.6";

            src = ./.;

@@ -2,8 +2,8 @@ apiVersion: v2
 name: rustfs
 description: RustFS helm chart to deploy RustFS on kubernetes cluster.
 type: application
-version: "0.4.0"
-appVersion: "1.0.0-beta.4"
+version: "0.6.0"
+appVersion: "1.0.0-beta.6"
 home: https://rustfs.com
 icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg
 maintainers:
@@ -51,6 +51,18 @@ spec:
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 10800
+  {{- else if eq $serviceType "LoadBalancer" }}
+  type: LoadBalancer
+  {{- with .Values.service.loadBalancerIP }}
+  loadBalancerIP: {{ . }}
+  {{- end }}
+  {{- with .Values.service.loadBalancerClass }}
+  loadBalancerClass: {{ . }}
+  {{- end }}
+  {{- with .Values.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
  {{- end }}
  ports:
    - name: endpoint
@@ -176,6 +176,10 @@ service:
    traefik.ingress.kubernetes.io/service.sticky.cookie.samesite: none
    traefik.ingress.kubernetes.io/service.sticky.cookie.secure: "true"
  type: ClusterIP
+  # LoadBalancer-specific fields (only used when type=LoadBalancer)
+  loadBalancerIP: ""           # Request a specific IP from the load balancer (e.g. for Cilium LB-IPAM or MetalLB)
+  loadBalancerClass: ""        # Specify a load balancer implementation (e.g. "io.cilium/bgp", "metallb")
+  loadBalancerSourceRanges: [] # Restrict client IPs (e.g. ["10.0.0.0/8"])
  endpoint:
    port: 9000
    nodePort: 32000
@@ -2,7 +2,7 @@
 %global _empty_manifest_terminate_build 0
 Name:           rustfs
 Version:        1.0.0
-Release:        beta.4
+Release:        beta.6
 Summary:       High-performance distributed object storage for MinIO alternative

 License:        Apache-2.0
@@ -57,6 +57,9 @@ install %_builddir/%{name}-%{version}-%{release}/target/%_arch/%_arch-unknown-li
 %_bindir/rustfs

 %changelog
+* Thu May 28 2026 houseme <housemecn@gmail.com>
+- Update RPM package to RustFS 1.0.0-beta.6
+
 * Thu May 20 2026 houseme <housemecn@gmail.com>
 - Update RPM package to RustFS 1.0.0-beta.4

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::admin::handlers::health::{HealthProbe, build_health_payload, collect_dependency_readiness, health_check_state};
+use crate::admin::handlers::health::{HealthProbe, build_health_response_parts, collect_dependency_readiness};
 use crate::license::has_valid_license;
 use crate::server::{CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, LICENSE, RUSTFS_ADMIN_PREFIX, VERSION};
 use crate::version::build;
@@ -28,7 +28,6 @@ use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri};
 use mime_guess::from_path;
 use rust_embed::RustEmbed;
 use serde::Serialize;
-use serde_json::json;
 use std::{
    net::{IpAddr, SocketAddr},
    sync::OnceLock,
@@ -283,7 +282,7 @@ async fn version_handler() -> impl IntoResponse {
            .header("content-type", "application/json")
            .status(StatusCode::OK)
            .body(Body::from(
-                json!({
+                serde_json::json!({
                    "version": cfg.release.version,
                    "version_info": cfg.version_info(),
                    "date": cfg.release.date,
@@ -466,6 +465,10 @@ fn setup_console_middleware_stack(
    if rustfs_utils::get_env_bool(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, rustfs_config::DEFAULT_HEALTH_ENDPOINT_ENABLE) {
        app = app
            .route(&format!("{CONSOLE_PREFIX}{HEALTH_PREFIX}"), get(health_check).head(health_check))
+            .route(
+                &format!("{CONSOLE_PREFIX}{}", crate::server::HEALTH_COMPAT_LIVE_PATH),
+                get(health_check).head(health_check),
+            )
            .route(&format!("{CONSOLE_PREFIX}{HEALTH_READY_PATH}"), get(health_check).head(health_check));
    } else {
        // Keep disabled health probes from falling through to the SPA fallback.
@@ -474,6 +477,10 @@ fn setup_console_middleware_stack(
                &format!("{CONSOLE_PREFIX}{HEALTH_PREFIX}"),
                get(health_route_disabled).head(health_route_disabled),
            )
+            .route(
+                &format!("{CONSOLE_PREFIX}{}", crate::server::HEALTH_COMPAT_LIVE_PATH),
+                get(health_route_disabled).head(health_route_disabled),
+            )
            .route(
                &format!("{CONSOLE_PREFIX}{HEALTH_READY_PATH}"),
                get(health_route_disabled).head(health_route_disabled),
@@ -523,21 +530,27 @@ async fn health_check(method: Method, uri: Uri) -> Response {
    } else {
        HealthProbe::Liveness
    };
-    let (storage_ready, iam_ready) = collect_dependency_readiness().await;
-    let health = health_check_state(storage_ready, iam_ready, probe);
+    let readiness_report = collect_dependency_readiness().await;
+    let uptime = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs();
+    let response_parts =
+        build_health_response_parts(method.clone(), probe, &readiness_report, "rustfs-console", Some(uptime), None);

    let builder = Response::builder()
-        .status(health.status_code)
+        .status(response_parts.status_code)
        .header("content-type", "application/json");

    match method {
        // GET: Returns complete JSON
        Method::GET => {
-            let uptime = std::time::SystemTime::now()
-                .duration_since(std::time::UNIX_EPOCH)
-                .unwrap_or_default()
-                .as_secs();
-            let body_json = build_health_payload(health, storage_ready, iam_ready, "rustfs-console", Some(uptime));
+            let body_json = response_parts.payload.unwrap_or_else(|| {
+                serde_json::json!({
+                    "status": "error",
+                    "service": "rustfs-console",
+                })
+            });

            // Return a minimal JSON when serialization fails to avoid panic
            let body_str = serde_json::to_string(&body_json).unwrap_or_else(|e| {
@@ -16,7 +16,7 @@ use super::profile::{TriggerProfileCPU, TriggerProfileMemory};
 use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::server::{
    HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH,
-    collect_dependency_readiness as collect_runtime_dependency_readiness,
+    collect_dependency_readiness_report as collect_runtime_dependency_readiness_report,
 };
 use http::{HeaderMap, HeaderValue};
 use hyper::{Method, StatusCode};
@@ -57,13 +57,35 @@ pub(crate) enum HealthProbe {
    Readiness,
 }

-pub(crate) async fn collect_dependency_readiness() -> (bool, bool) {
-    let readiness = collect_runtime_dependency_readiness().await;
-    (readiness.storage_ready, readiness.iam_ready)
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct HealthResponseParts {
+    pub(crate) status_code: StatusCode,
+    pub(crate) payload: Option<Value>,
 }

-pub(crate) fn health_check_state(storage_ready: bool, iam_ready: bool, probe: HealthProbe) -> HealthCheckState {
-    let ready = storage_ready && iam_ready;
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct HealthPayloadContext<'a> {
+    pub(crate) health: HealthCheckState,
+    pub(crate) storage_ready: bool,
+    pub(crate) iam_ready: bool,
+    pub(crate) lock_quorum_ready: bool,
+    pub(crate) degraded_reasons: &'a [crate::server::ReadinessDegradedReason],
+    pub(crate) service: &'a str,
+    pub(crate) uptime: Option<u64>,
+    pub(crate) kms_ready: Option<bool>,
+}
+
+pub(crate) async fn collect_dependency_readiness() -> crate::server::DependencyReadinessReport {
+    collect_runtime_dependency_readiness_report().await
+}
+
+pub(crate) fn health_check_state(
+    storage_ready: bool,
+    iam_ready: bool,
+    lock_quorum_ready: bool,
+    probe: HealthProbe,
+) -> HealthCheckState {
+    let ready = storage_ready && iam_ready && lock_quorum_ready;
    let status = if ready { "ok" } else { "degraded" };

    let status_code = match probe {
@@ -86,8 +108,13 @@ pub(crate) fn health_minimal_response_enabled() -> bool {
    )
 }

-pub(crate) fn build_component_details(storage_ready: bool, iam_ready: bool) -> Value {
-    json!({
+pub(crate) fn build_component_details(
+    storage_ready: bool,
+    iam_ready: bool,
+    lock_quorum_ready: bool,
+    kms_ready: Option<bool>,
+) -> Value {
+    let mut details = json!({
        "storage": {
            "status": if storage_ready { "connected" } else { "disconnected" },
            "ready": storage_ready,
@@ -95,8 +122,30 @@ pub(crate) fn build_component_details(storage_ready: bool, iam_ready: bool) -> V
        "iam": {
            "status": if iam_ready { "connected" } else { "disconnected" },
            "ready": iam_ready,
+        },
+        "lock": {
+            "status": if lock_quorum_ready { "connected" } else { "disconnected" },
+            "ready": lock_quorum_ready,
        }
-    })
+    });
+
+    if let Some(kms_ready) = kms_ready {
+        details["kms"] = json!({
+            "status": if kms_ready { "connected" } else { "disconnected" },
+            "ready": kms_ready,
+        });
+    }
+
+    details
+}
+
+pub(crate) fn build_degraded_reasons(reasons: &[crate::server::ReadinessDegradedReason]) -> Value {
+    Value::Array(
+        reasons
+            .iter()
+            .map(|reason| Value::String(reason.as_str().to_string()))
+            .collect(),
+    )
 }

 pub(crate) fn probe_from_path(path: &str) -> HealthProbe {
@@ -107,58 +156,77 @@ pub(crate) fn probe_from_path(path: &str) -> HealthProbe {
    }
 }

-pub(crate) fn build_health_payload(
-    health: HealthCheckState,
-    storage_ready: bool,
-    iam_ready: bool,
+pub(crate) fn build_health_response_parts(
+    method: Method,
+    probe: HealthProbe,
+    readiness_report: &crate::server::DependencyReadinessReport,
    service: &str,
    uptime: Option<u64>,
-) -> Value {
+    kms_ready: Option<bool>,
+) -> HealthResponseParts {
+    let storage_ready = readiness_report.readiness.storage_ready;
+    let iam_ready = readiness_report.readiness.iam_ready;
+    let lock_quorum_ready = readiness_report.readiness.lock_quorum_ready;
+    let mut health = health_check_state(storage_ready, iam_ready, lock_quorum_ready, probe);
+    let mut degraded_reasons = readiness_report.degraded_reasons.clone();
+
+    if probe == HealthProbe::Readiness && matches!(kms_ready, Some(false)) {
+        health = HealthCheckState {
+            status_code: StatusCode::SERVICE_UNAVAILABLE,
+            status: "degraded",
+            ready: false,
+        };
+        if !degraded_reasons.contains(&crate::server::ReadinessDegradedReason::KmsNotReady) {
+            degraded_reasons.push(crate::server::ReadinessDegradedReason::KmsNotReady);
+        }
+    }
+
+    let payload = if method == Method::HEAD {
+        None
+    } else {
+        Some(build_health_payload(HealthPayloadContext {
+            health,
+            storage_ready,
+            iam_ready,
+            lock_quorum_ready,
+            degraded_reasons: &degraded_reasons,
+            service,
+            uptime,
+            kms_ready,
+        }))
+    };
+
+    HealthResponseParts {
+        status_code: health.status_code,
+        payload,
+    }
+}
+
+pub(crate) fn build_health_payload(ctx: HealthPayloadContext<'_>) -> Value {
    if health_minimal_response_enabled() {
        return json!({
-            "status": health.status,
-            "ready": health.ready,
+            "status": ctx.health.status,
+            "ready": ctx.health.ready,
        });
    }

    let mut payload = json!({
-        "status": health.status,
-        "ready": health.ready,
-        "service": service,
+        "status": ctx.health.status,
+        "ready": ctx.health.ready,
+        "service": ctx.service,
        "timestamp": jiff::Zoned::now().to_string(),
        "version": env!("CARGO_PKG_VERSION"),
-        "details": build_component_details(storage_ready, iam_ready),
+        "details": build_component_details(ctx.storage_ready, ctx.iam_ready, ctx.lock_quorum_ready, ctx.kms_ready),
+        "degradedReasons": build_degraded_reasons(ctx.degraded_reasons),
    });

-    if let Some(uptime) = uptime {
+    if let Some(uptime) = ctx.uptime {
        payload["uptime"] = json!(uptime);
    }

    payload
 }

-pub(crate) fn build_health_response(
-    method: Method,
-    probe: HealthProbe,
-    storage_ready: bool,
-    iam_ready: bool,
-) -> S3Response<(StatusCode, Body)> {
-    let health = health_check_state(storage_ready, iam_ready, probe);
-    let health_info = build_health_payload(health, storage_ready, iam_ready, "rustfs-endpoint", None);
-
-    let mut headers = HeaderMap::new();
-    headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
-
-    if method == Method::HEAD {
-        return S3Response::with_headers((health.status_code, Body::empty()), headers);
-    }
-
-    let body_str = serde_json::to_string(&health_info).unwrap_or_else(|_| "{}".to_string());
-    let body = Body::from(body_str);
-
-    S3Response::with_headers((health.status_code, body), headers)
-}
-
 #[async_trait::async_trait]
 impl Operation for HealthCheckHandler {
    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
@@ -177,9 +245,21 @@ impl Operation for HealthCheckHandler {
        }

        let probe = probe_from_path(req.uri.path());
-        let (storage_ready, iam_ready) = collect_dependency_readiness().await;
+        let readiness_report = collect_dependency_readiness().await;

-        Ok(build_health_response(method, probe, storage_ready, iam_ready))
+        let response_parts = build_health_response_parts(method.clone(), probe, &readiness_report, "rustfs-endpoint", None, None);
+
+        let mut headers = HeaderMap::new();
+        headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+
+        let response = if let Some(payload) = response_parts.payload {
+            let body_str = serde_json::to_string(&payload).unwrap_or_else(|_| "{}".to_string());
+            S3Response::with_headers((response_parts.status_code, Body::from(body_str)), headers)
+        } else {
+            S3Response::with_headers((response_parts.status_code, Body::empty()), headers)
+        };
+
+        Ok(response)
    }
 }

@@ -190,7 +270,7 @@ mod tests {

    #[test]
    fn test_readiness_state_ready() {
-        let state = health_check_state(true, true, HealthProbe::Readiness);
+        let state = health_check_state(true, true, true, HealthProbe::Readiness);
        assert_eq!(state.status_code, StatusCode::OK);
        assert_eq!(state.status, "ok");
        assert!(state.ready);
@@ -198,7 +278,7 @@ mod tests {

    #[test]
    fn test_readiness_state_storage_not_ready() {
-        let state = health_check_state(false, true, HealthProbe::Readiness);
+        let state = health_check_state(false, true, true, HealthProbe::Readiness);
        assert_eq!(state.status_code, StatusCode::SERVICE_UNAVAILABLE);
        assert_eq!(state.status, "degraded");
        assert!(!state.ready);
@@ -206,7 +286,7 @@ mod tests {

    #[test]
    fn test_liveness_state_iam_not_ready() {
-        let state = health_check_state(true, false, HealthProbe::Liveness);
+        let state = health_check_state(true, false, true, HealthProbe::Liveness);
        assert_eq!(state.status_code, StatusCode::OK);
        assert_eq!(state.status, "degraded");
        assert!(!state.ready);
@@ -214,7 +294,15 @@ mod tests {

    #[test]
    fn test_readiness_state_iam_not_ready() {
-        let state = health_check_state(true, false, HealthProbe::Readiness);
+        let state = health_check_state(true, false, true, HealthProbe::Readiness);
+        assert_eq!(state.status_code, StatusCode::SERVICE_UNAVAILABLE);
+        assert_eq!(state.status, "degraded");
+        assert!(!state.ready);
+    }
+
+    #[test]
+    fn test_readiness_state_lock_not_ready() {
+        let state = health_check_state(true, true, false, HealthProbe::Readiness);
        assert_eq!(state.status_code, StatusCode::SERVICE_UNAVAILABLE);
        assert_eq!(state.status, "degraded");
        assert!(!state.ready);
@@ -222,12 +310,22 @@ mod tests {

    #[test]
    fn test_health_check_component_details() {
-        let details = build_component_details(true, false);
+        let details = build_component_details(true, false, false, None);

        assert_eq!(details["storage"]["status"], "connected");
        assert_eq!(details["storage"]["ready"], true);
        assert_eq!(details["iam"]["status"], "disconnected");
        assert_eq!(details["iam"]["ready"], false);
+        assert_eq!(details["lock"]["status"], "disconnected");
+        assert_eq!(details["lock"]["ready"], false);
+        assert!(details.get("kms").is_none());
+    }
+
+    #[test]
+    fn test_health_check_component_details_include_kms_when_present() {
+        let details = build_component_details(true, true, true, Some(false));
+        assert_eq!(details["kms"]["status"], "disconnected");
+        assert_eq!(details["kms"]["ready"], false);
    }

    #[test]
@@ -243,33 +341,79 @@ mod tests {

    #[test]
    fn test_build_health_response_readiness_returns_503_when_deps_not_ready() {
-        let resp = build_health_response(Method::GET, HealthProbe::Readiness, false, true);
-        assert_eq!(resp.output.0, StatusCode::SERVICE_UNAVAILABLE);
+        let readiness_report = crate::server::DependencyReadinessReport {
+            readiness: crate::server::DependencyReadiness {
+                storage_ready: false,
+                iam_ready: true,
+                lock_quorum_ready: true,
+            },
+            degraded_reasons: vec![crate::server::ReadinessDegradedReason::StorageQuorumUnavailable],
+        };
+        let parts =
+            build_health_response_parts(Method::GET, HealthProbe::Readiness, &readiness_report, "rustfs-endpoint", None, None);
+        assert_eq!(parts.status_code, StatusCode::SERVICE_UNAVAILABLE);
    }

    #[test]
    fn test_build_health_response_readiness_returns_200_when_deps_ready() {
-        let resp = build_health_response(Method::GET, HealthProbe::Readiness, true, true);
-        assert_eq!(resp.output.0, StatusCode::OK);
+        let readiness_report = crate::server::DependencyReadinessReport {
+            readiness: crate::server::DependencyReadiness {
+                storage_ready: true,
+                iam_ready: true,
+                lock_quorum_ready: true,
+            },
+            degraded_reasons: Vec::new(),
+        };
+        let parts =
+            build_health_response_parts(Method::GET, HealthProbe::Readiness, &readiness_report, "rustfs-endpoint", None, None);
+        assert_eq!(parts.status_code, StatusCode::OK);
    }

    #[test]
    fn test_build_health_response_liveness_returns_200_when_deps_not_ready() {
-        let resp = build_health_response(Method::GET, HealthProbe::Liveness, false, false);
-        assert_eq!(resp.output.0, StatusCode::OK);
+        let readiness_report = crate::server::DependencyReadinessReport {
+            readiness: crate::server::DependencyReadiness {
+                storage_ready: false,
+                iam_ready: false,
+                lock_quorum_ready: false,
+            },
+            degraded_reasons: vec![crate::server::ReadinessDegradedReason::StorageAndIamUnavailable],
+        };
+        let parts =
+            build_health_response_parts(Method::GET, HealthProbe::Liveness, &readiness_report, "rustfs-endpoint", None, None);
+        assert_eq!(parts.status_code, StatusCode::OK);
    }

    #[test]
    fn test_build_health_response_head_returns_empty_body() {
-        let resp = build_health_response(Method::HEAD, HealthProbe::Readiness, false, false);
-        assert_eq!(resp.output.0, StatusCode::SERVICE_UNAVAILABLE);
+        let readiness_report = crate::server::DependencyReadinessReport {
+            readiness: crate::server::DependencyReadiness {
+                storage_ready: false,
+                iam_ready: false,
+                lock_quorum_ready: false,
+            },
+            degraded_reasons: vec![crate::server::ReadinessDegradedReason::StorageAndIamUnavailable],
+        };
+        let parts =
+            build_health_response_parts(Method::HEAD, HealthProbe::Readiness, &readiness_report, "rustfs-endpoint", None, None);
+        assert_eq!(parts.status_code, StatusCode::SERVICE_UNAVAILABLE);
+        assert!(parts.payload.is_none());
    }

    #[test]
    fn test_build_health_payload_minimal_mode_returns_status_and_ready_only() {
-        let health = health_check_state(true, false, HealthProbe::Readiness);
+        let health = health_check_state(true, false, true, HealthProbe::Readiness);
        with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("true"), || {
-            let payload = build_health_payload(health, true, false, "rustfs-endpoint", Some(123));
+            let payload = build_health_payload(HealthPayloadContext {
+                health,
+                storage_ready: true,
+                iam_ready: false,
+                lock_quorum_ready: true,
+                degraded_reasons: &[crate::server::ReadinessDegradedReason::IamNotReady],
+                service: "rustfs-endpoint",
+                uptime: Some(123),
+                kms_ready: None,
+            });
            assert_eq!(payload["status"], "degraded");
            assert_eq!(payload["ready"], false);
            assert!(payload.get("version").is_none());
@@ -278,4 +422,73 @@ mod tests {
            assert!(payload.get("uptime").is_none());
        });
    }
+
+    #[test]
+    fn test_build_health_payload_includes_degraded_reasons() {
+        let health = health_check_state(false, false, false, HealthProbe::Readiness);
+        let payload = build_health_payload(HealthPayloadContext {
+            health,
+            storage_ready: false,
+            iam_ready: false,
+            lock_quorum_ready: false,
+            degraded_reasons: &[crate::server::ReadinessDegradedReason::StorageAndIamUnavailable],
+            service: "rustfs-endpoint",
+            uptime: None,
+            kms_ready: None,
+        });
+        assert_eq!(payload["degradedReasons"][0], "storage_and_iam_unavailable");
+    }
+
+    #[test]
+    fn test_build_health_response_parts_head_has_no_payload() {
+        let report = crate::server::DependencyReadinessReport {
+            readiness: crate::server::DependencyReadiness {
+                storage_ready: true,
+                iam_ready: true,
+                lock_quorum_ready: true,
+            },
+            degraded_reasons: Vec::new(),
+        };
+        let parts = build_health_response_parts(Method::HEAD, HealthProbe::Readiness, &report, "rustfs-endpoint", None, None);
+        assert_eq!(parts.status_code, StatusCode::OK);
+        assert!(parts.payload.is_none());
+    }
+
+    #[test]
+    fn test_build_health_response_parts_get_includes_payload() {
+        let report = crate::server::DependencyReadinessReport {
+            readiness: crate::server::DependencyReadiness {
+                storage_ready: false,
+                iam_ready: true,
+                lock_quorum_ready: true,
+            },
+            degraded_reasons: vec![crate::server::ReadinessDegradedReason::StorageQuorumUnavailable],
+        };
+        let parts = build_health_response_parts(Method::GET, HealthProbe::Readiness, &report, "rustfs-endpoint", None, None);
+        assert_eq!(parts.status_code, StatusCode::SERVICE_UNAVAILABLE);
+        let payload = parts.payload.expect("GET should include payload");
+        assert_eq!(payload["status"], "degraded");
+        assert_eq!(payload["ready"], false);
+        assert_eq!(payload["degradedReasons"][0], "storage_quorum_unavailable");
+    }
+
+    #[test]
+    fn test_build_health_response_parts_readiness_marks_kms_not_ready() {
+        let report = crate::server::DependencyReadinessReport {
+            readiness: crate::server::DependencyReadiness {
+                storage_ready: true,
+                iam_ready: true,
+                lock_quorum_ready: true,
+            },
+            degraded_reasons: Vec::new(),
+        };
+        let parts =
+            build_health_response_parts(Method::GET, HealthProbe::Readiness, &report, "rustfs-endpoint", None, Some(false));
+        assert_eq!(parts.status_code, StatusCode::SERVICE_UNAVAILABLE);
+        let payload = parts.payload.expect("GET should include payload");
+        assert_eq!(payload["ready"], false);
+        assert_eq!(payload["details"]["lock"]["ready"], true);
+        assert_eq!(payload["details"]["kms"]["ready"], false);
+        assert_eq!(payload["degradedReasons"][0], "kms_not_ready");
+    }
 }
@@ -64,7 +64,7 @@ use rustfs_policy::policy::{
 };
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
-use rustfs_tls_runtime::load_global_outbound_tls_state;
+use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, load_global_outbound_tls_generation, load_global_outbound_tls_state};
 use rustfs_utils::http::get_source_scheme;
 use rustls_pki_types::pem::PemObject;
 use s3s::dto::{
@@ -79,9 +79,10 @@ use serde::de::DeserializeOwned;
 use serde_json::Value;
 use sha2::{Digest, Sha256};
 use std::collections::{BTreeMap, HashMap, HashSet};
+use std::sync::LazyLock;
 use std::time::{Duration, Instant};
 use time::OffsetDateTime;
-use tokio::sync::OnceCell;
+use tokio::sync::Mutex;
 use tracing::warn;
 use url::{Url, form_urlencoded};
 use uuid::Uuid;
@@ -102,7 +103,36 @@ const SITE_REPLICATOR_SERVICE_ACCOUNT: &str = "site-replicator-0";
 const SITE_REPLICATION_PEER_JOIN_PATH: &str = "/rustfs/admin/v3/site-replication/peer/join";
 const SITE_REPLICATION_PEER_EDIT_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit";
 const SITE_REPLICATION_PEER_REMOVE_PATH: &str = "/rustfs/admin/v3/site-replication/peer/remove";
-static SITE_REPLICATION_PEER_CLIENT: OnceCell<Result<reqwest::Client, String>> = OnceCell::const_new();
+#[derive(Clone)]
+enum SiteReplicationPeerClientCacheEntry {
+    Ready(reqwest::Client),
+    Failed(String),
+}
+
+#[derive(Clone)]
+struct SiteReplicationPeerClientCache {
+    generation: u64,
+    entry: SiteReplicationPeerClientCacheEntry,
+}
+
+static SITE_REPLICATION_PEER_CLIENT: LazyLock<Mutex<Option<SiteReplicationPeerClientCache>>> = LazyLock::new(|| Mutex::new(None));
+
+fn site_replication_peer_client_cache_hit(
+    cache: &Option<SiteReplicationPeerClientCache>,
+    generation: u64,
+) -> Option<S3Result<reqwest::Client>> {
+    let cached = cache.as_ref()?;
+    if cached.generation != generation {
+        return None;
+    }
+    Some(match &cached.entry {
+        SiteReplicationPeerClientCacheEntry::Ready(client) => Ok(client.clone()),
+        SiteReplicationPeerClientCacheEntry::Failed(err) => Err(S3Error::with_message(
+            S3ErrorCode::InternalError,
+            format!("initialize site replication peer client failed: {err}"),
+        )),
+    })
+}

 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 struct SiteReplicationState {
@@ -447,13 +477,12 @@ async fn persist_site_replication_state(state: &SiteReplicationState) -> S3Resul
    }
 }

-async fn build_site_replication_peer_client() -> S3Result<reqwest::Client> {
+fn build_site_replication_peer_client(outbound_tls: &GlobalPublishedOutboundTlsState) -> S3Result<reqwest::Client> {
    let mut builder = reqwest::Client::builder()
        .timeout(SITE_REPLICATION_PEER_REQUEST_TIMEOUT)
        .connect_timeout(SITE_REPLICATION_PEER_CONNECT_TIMEOUT)
        .pool_idle_timeout(Some(Duration::from_secs(60)));

-    let outbound_tls = load_global_outbound_tls_state().await;
    if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
        let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
        let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
@@ -481,16 +510,30 @@ async fn build_site_replication_peer_client() -> S3Result<reqwest::Client> {
        .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build site replication peer client failed: {e}")))
 }

-async fn site_replication_peer_client() -> S3Result<&'static reqwest::Client> {
-    let result = SITE_REPLICATION_PEER_CLIENT
-        .get_or_init(|| async { build_site_replication_peer_client().await.map_err(|e| e.to_string()) })
-        .await;
-    result.as_ref().map_err(|err| {
-        S3Error::with_message(
-            S3ErrorCode::InternalError,
-            format!("initialize site replication peer client failed: {err}"),
-        )
-    })
+async fn site_replication_peer_client() -> S3Result<reqwest::Client> {
+    let generation = load_global_outbound_tls_generation().0;
+    let cache = SITE_REPLICATION_PEER_CLIENT.lock().await;
+    if let Some(hit) = site_replication_peer_client_cache_hit(&cache, generation) {
+        return hit;
+    }
+    drop(cache);
+
+    let outbound_tls = load_global_outbound_tls_state().await;
+    let built = build_site_replication_peer_client(&outbound_tls);
+    let cache_entry = match &built {
+        Ok(client) => SiteReplicationPeerClientCacheEntry::Ready(client.clone()),
+        Err(err) => SiteReplicationPeerClientCacheEntry::Failed(err.to_string()),
+    };
+
+    let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await;
+    if cache.as_ref().is_none_or(|cached| cached.generation <= generation) {
+        *cache = Some(SiteReplicationPeerClientCache {
+            generation,
+            entry: cache_entry,
+        });
+    }
+
+    built
 }

 fn runtime_tls_enabled() -> bool {
@@ -2986,6 +3029,8 @@ impl Operation for SRStateEditHandler {
 mod tests {
    use super::*;
    use http::{HeaderMap, HeaderValue, Uri};
+    use rustfs_common::{get_global_outbound_tls_generation, set_global_outbound_tls_generation};
+    use serial_test::serial;
    use temp_env::with_var;

    fn peer(name: &str, endpoint: &str) -> PeerInfo {
@@ -3724,6 +3769,77 @@ mod tests {
        assert!(ldap_configs.configs.contains_key("default"));
    }

+    #[test]
+    fn test_site_replication_peer_client_cache_hit_generation_mismatch_returns_none() {
+        let cache = Some(SiteReplicationPeerClientCache {
+            generation: 7,
+            entry: SiteReplicationPeerClientCacheEntry::Failed("cached error".to_string()),
+        });
+
+        assert!(site_replication_peer_client_cache_hit(&cache, 8).is_none());
+    }
+
+    #[test]
+    fn test_site_replication_peer_client_cache_hit_returns_cached_ready_client() {
+        let cache = Some(SiteReplicationPeerClientCache {
+            generation: 7,
+            entry: SiteReplicationPeerClientCacheEntry::Ready(reqwest::Client::new()),
+        });
+
+        site_replication_peer_client_cache_hit(&cache, 7)
+            .expect("cache hit expected")
+            .expect("ready cache entry should return cached client");
+    }
+
+    #[test]
+    fn test_site_replication_peer_client_cache_hit_returns_cached_error() {
+        let cache = Some(SiteReplicationPeerClientCache {
+            generation: 7,
+            entry: SiteReplicationPeerClientCacheEntry::Failed("cached error".to_string()),
+        });
+
+        let err = site_replication_peer_client_cache_hit(&cache, 7)
+            .expect("cache hit expected")
+            .expect_err("error cache entry should return error");
+        assert!(err.to_string().contains("cached error"), "expected cached error detail, got: {}", err);
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_site_replication_peer_client_rebuilds_when_generation_changes() {
+        let previous_generation = get_global_outbound_tls_generation();
+        let previous_cache = {
+            let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await;
+            let snapshot = cache.clone();
+            *cache = None;
+            snapshot
+        };
+
+        set_global_outbound_tls_generation(101);
+        site_replication_peer_client()
+            .await
+            .expect("initial client build should succeed");
+        let cache = SITE_REPLICATION_PEER_CLIENT.lock().await;
+        let cached = cache.as_ref().expect("cache should be populated");
+        assert_eq!(cached.generation, 101);
+        assert!(matches!(cached.entry, SiteReplicationPeerClientCacheEntry::Ready(_)));
+        drop(cache);
+
+        set_global_outbound_tls_generation(102);
+        site_replication_peer_client()
+            .await
+            .expect("new generation should rebuild client");
+        let cache = SITE_REPLICATION_PEER_CLIENT.lock().await;
+        let cached = cache.as_ref().expect("cache should be populated");
+        assert_eq!(cached.generation, 102);
+        assert!(matches!(cached.entry, SiteReplicationPeerClientCacheEntry::Ready(_)));
+
+        drop(cache);
+        set_global_outbound_tls_generation(previous_generation);
+        let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await;
+        *cache = previous_cache;
+    }
+
    #[test]
    fn test_gob_site_netperf_node_result_matches_go_encoding() {
        let data = encode_go_gob_site_netperf_node_result(&SiteNetPerfNodeResult {
@@ -26,7 +26,7 @@ use crate::storage::options::{
 };
 use crate::storage::s3_api::multipart::{
    ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output,
-    parse_list_multipart_uploads_params, parse_list_parts_params,
+    parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
 };
 use crate::storage::sse::{build_ssec_read_headers, encryption_material_to_metadata, map_get_object_reader_error};
 use crate::storage::*;
@@ -668,7 +668,7 @@ impl DefaultMultipartUsecase {
            ..
        } = input;

-        let part_id = part_number as usize;
+        let part_id = parse_upload_part_number(part_number)?;

        let mut size = content_length;
        let mut body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?;
@@ -1006,7 +1006,7 @@ impl DefaultMultipartUsecase {
            None
        };

-        let part_id = part_number as usize;
+        let part_id = parse_upload_part_number(part_number)?;

        let Some(store) = new_object_layer_fn() else {
            return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
@@ -1726,6 +1726,29 @@ mod tests {
        assert_eq!(err.code(), &S3ErrorCode::InternalError);
    }

+    #[tokio::test]
+    async fn execute_upload_part_copy_rejects_invalid_part_number_before_store_lookup() {
+        for part_number in [-1, 0, 10001] {
+            let input = UploadPartCopyInput::builder()
+                .bucket("bucket".to_string())
+                .key("object".to_string())
+                .copy_source(CopySource::Bucket {
+                    bucket: "src-bucket".into(),
+                    key: "src-object".into(),
+                    version_id: None,
+                })
+                .part_number(part_number)
+                .upload_id("upload-id".to_string())
+                .build()
+                .unwrap();
+            let req = build_request(input, Method::PUT);
+
+            let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
+            assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+            assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
+        }
+    }
+
    #[test]
    fn test_validate_copy_source_range_not_exceeds_returns_invalid_range_when_range_exceeds() {
        use super::validate_copy_source_range_not_exceeds;
@@ -1778,4 +1801,22 @@ mod tests {
        let err = make_usecase().execute_upload_part(req).await.unwrap_err();
        assert_eq!(err.code(), &S3ErrorCode::IncompleteBody);
    }
+
+    #[tokio::test]
+    async fn execute_upload_part_rejects_invalid_part_number_before_body_lookup() {
+        for part_number in [-1, 0, 10001] {
+            let input = UploadPartInput::builder()
+                .bucket("bucket".to_string())
+                .key("object".to_string())
+                .upload_id("upload-id".to_string())
+                .part_number(part_number)
+                .build()
+                .unwrap();
+            let req = build_request(input, Method::PUT);
+
+            let err = make_usecase().execute_upload_part(req).await.unwrap_err();
+            assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+            assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
+        }
+    }
 }
@@ -110,7 +110,7 @@ use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
 use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
 use std::collections::HashMap;
 use std::ops::Add;
-use std::path::Path;
+use std::path::{Component, Path};

 use std::str::FromStr;
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -775,16 +775,36 @@ fn snowball_meta_flag(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &s
    snowball_meta_value(headers, exact_keys, suffix_lower).is_some_and(|value| value.eq_ignore_ascii_case("true"))
 }

-fn normalize_snowball_prefix(prefix: &str) -> Option<String> {
-    let normalized = prefix.trim().trim_matches('/');
-    if normalized.is_empty() {
-        return None;
-    }
-
-    Some(normalized.to_string())
+fn contains_parent_dir_component(path: &str) -> bool {
+    path.split(['/', '\\']).any(|component| component == "..")
 }

-fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -> String {
+fn validate_extract_relative_path(path: &str) -> S3Result<()> {
+    let path = Path::new(path);
+    if path
+        .components()
+        .any(|component| matches!(component, Component::Prefix(_) | Component::RootDir | Component::ParentDir))
+        || contains_parent_dir_component(path.to_string_lossy().as_ref())
+    {
+        return Err(s3_error!(InvalidArgument, "archive entry path must stay within the target bucket"));
+    }
+
+    Ok(())
+}
+
+fn normalize_snowball_prefix(prefix: &str) -> S3Result<Option<String>> {
+    let normalized = prefix.trim().trim_matches('/');
+    if normalized.is_empty() {
+        return Ok(None);
+    }
+
+    validate_extract_relative_path(normalized)?;
+
+    Ok(Some(normalized.to_string()))
+}
+
+fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -> S3Result<String> {
+    validate_extract_relative_path(path)?;
    let path = path.trim_matches('/');
    let mut key = match prefix {
        Some(prefix) if !path.is_empty() => format!("{prefix}/{path}"),
@@ -796,7 +816,9 @@ fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -
        key.push('/');
    }

-    key
+    validate_extract_relative_path(&key)?;
+
+    Ok(key)
 }

 fn map_extract_archive_error(err: impl std::fmt::Display) -> S3Error {
@@ -992,17 +1014,19 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool {
    opts.version_id.is_none() && opts.versioned && !opts.version_suspended
 }

-fn resolve_put_object_extract_options(headers: &HeaderMap) -> PutObjectExtractOptions {
+fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result<PutObjectExtractOptions> {
    let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER)
-        .and_then(|value| normalize_snowball_prefix(&value));
+        .map(|value| normalize_snowball_prefix(&value))
+        .transpose()?
+        .flatten();
    let ignore_dirs = snowball_meta_flag(headers, SNOWBALL_IGNORE_DIRS_HEADER_KEYS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER);
    let ignore_errors = snowball_meta_flag(headers, SNOWBALL_IGNORE_ERRORS_HEADER_KEYS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER);

-    PutObjectExtractOptions {
+    Ok(PutObjectExtractOptions {
        prefix,
        ignore_dirs,
        ignore_errors,
-    }
+    })
 }

 fn is_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool {
@@ -4126,7 +4150,7 @@ impl DefaultObjectUsecase {
            return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
        };

-        let extract_options = resolve_put_object_extract_options(&req.headers);
+        let extract_options = resolve_put_object_extract_options(&req.headers)?;
        let version_id = match event_version_id {
            Some(v) => v.to_string(),
            None => String::new(),
@@ -4167,7 +4191,16 @@ impl DefaultObjectUsecase {
            };

            let is_dir = f.header().entry_type().is_dir();
-            let fpath = normalize_extract_entry_key(&fpath.to_string_lossy(), extract_options.prefix.as_deref(), is_dir);
+            let fpath = match normalize_extract_entry_key(&fpath.to_string_lossy(), extract_options.prefix.as_deref(), is_dir) {
+                Ok(fpath) => fpath,
+                Err(err) => {
+                    if extract_options.ignore_errors {
+                        warn!("Skipping archive entry because path is unsafe and ignore-errors is enabled: {err}");
+                        continue;
+                    }
+                    return Err(err);
+                }
+            };

            let mut auth_req = S3Request {
                input: PutObjectInput::default(),
@@ -4464,18 +4497,39 @@ mod tests {

    #[test]
    fn normalize_snowball_prefix_trims_slashes_and_whitespace() {
-        assert_eq!(normalize_snowball_prefix(" /batch/incoming/ "), Some("batch/incoming".to_string()));
-        assert_eq!(normalize_snowball_prefix("///"), None);
+        assert_eq!(
+            normalize_snowball_prefix(" /batch/incoming/ ").unwrap(),
+            Some("batch/incoming".to_string())
+        );
+        assert_eq!(normalize_snowball_prefix("///").unwrap(), None);
+    }
+
+    #[test]
+    fn normalize_snowball_prefix_rejects_parent_dir_components() {
+        assert!(normalize_snowball_prefix("../victim-bucket").is_err());
+        assert!(normalize_snowball_prefix("safe/../../victim-bucket").is_err());
+        assert!(normalize_snowball_prefix("safe\\..\\victim-bucket").is_err());
    }

    #[test]
    fn normalize_extract_entry_key_applies_prefix_and_directory_suffix() {
        assert_eq!(
-            normalize_extract_entry_key("nested/path.txt", Some("imports"), false),
+            normalize_extract_entry_key("nested/path.txt", Some("imports"), false).unwrap(),
            "imports/nested/path.txt"
        );
-        assert_eq!(normalize_extract_entry_key("nested/dir/", Some("imports"), true), "imports/nested/dir/");
-        assert_eq!(normalize_extract_entry_key("top-level", None, false), "top-level");
+        assert_eq!(
+            normalize_extract_entry_key("nested/dir/", Some("imports"), true).unwrap(),
+            "imports/nested/dir/"
+        );
+        assert_eq!(normalize_extract_entry_key("top-level", None, false).unwrap(), "top-level");
+    }
+
+    #[test]
+    fn normalize_extract_entry_key_rejects_bucket_escape_paths() {
+        assert!(normalize_extract_entry_key("../victim-bucket/evil.txt", None, false).is_err());
+        assert!(normalize_extract_entry_key("safe/../../victim-bucket/evil.txt", None, false).is_err());
+        assert!(normalize_extract_entry_key("safe\\..\\victim-bucket\\evil.txt", None, false).is_err());
+        assert!(normalize_extract_entry_key("evil.txt", Some("../victim-bucket"), false).is_err());
    }

    #[test]
@@ -4700,7 +4754,7 @@ mod tests {
    #[test]
    fn resolve_put_object_extract_options_defaults_when_headers_missing() {
        let headers = HeaderMap::new();
-        let options = resolve_put_object_extract_options(&headers);
+        let options = resolve_put_object_extract_options(&headers).unwrap();
        assert_eq!(
            options,
            PutObjectExtractOptions {
@@ -4718,7 +4772,7 @@ mod tests {
        headers.insert(AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, HeaderValue::from_static("true"));
        headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, HeaderValue::from_static("TRUE"));

-        let options = resolve_put_object_extract_options(&headers);
+        let options = resolve_put_object_extract_options(&headers).unwrap();
        assert_eq!(options.prefix.as_deref(), Some("internal/prefix"));
        assert!(options.ignore_dirs);
        assert!(options.ignore_errors);
@@ -4731,7 +4785,7 @@ mod tests {
        headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static(" true "));
        headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("TRUE"));

-        let options = resolve_put_object_extract_options(&headers);
+        let options = resolve_put_object_extract_options(&headers).unwrap();
        assert_eq!(options.prefix.as_deref(), Some("standard/prefix"));
        assert!(options.ignore_dirs);
        assert!(options.ignore_errors);
@@ -4753,7 +4807,7 @@ mod tests {
            HeaderValue::from_static("TRUE"),
        );

-        let options = resolve_put_object_extract_options(&headers);
+        let options = resolve_put_object_extract_options(&headers).unwrap();
        assert_eq!(options.prefix.as_deref(), Some("partner/import"));
        assert!(options.ignore_dirs);
        assert!(options.ignore_errors);
@@ -4767,7 +4821,7 @@ mod tests {
        headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static("/standard/prefix/"));
        headers.insert(AMZ_MINIO_SNOWBALL_PREFIX, HeaderValue::from_static("/minio/prefix/"));

-        let options = resolve_put_object_extract_options(&headers);
+        let options = resolve_put_object_extract_options(&headers).unwrap();
        assert_eq!(options.prefix.as_deref(), Some("minio/prefix"));
    }

@@ -4779,11 +4833,19 @@ mod tests {
        headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("false"));
        headers.insert("x-amz-meta-acme-snowball-ignore-errors", HeaderValue::from_static("true"));

-        let options = resolve_put_object_extract_options(&headers);
+        let options = resolve_put_object_extract_options(&headers).unwrap();
        assert!(!options.ignore_dirs);
        assert!(!options.ignore_errors);
    }

+    #[test]
+    fn resolve_put_object_extract_options_rejects_unsafe_prefix_header() {
+        let mut headers = HeaderMap::new();
+        headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static("../victim-bucket"));
+
+        assert!(resolve_put_object_extract_options(&headers).is_err());
+    }
+
    #[tokio::test]
    async fn execute_put_object_rejects_post_object_sse_kms_from_input() {
        let input = PutObjectInput::builder()
@@ -16,9 +16,10 @@
 //!
 //! This module contains the command-line interface definitions including:
 //! - `Cli`: Main CLI parser
-//! - `Commands`: Subcommands (Server, Info)
+//! - `Commands`: Subcommands (Server, Info, Tls)
 //! - `ServerOpts`: Server subcommand options
 //! - `InfoOpts`: Info subcommand options
+//! - `TlsOpts`: TLS diagnostic subcommand options
 //! - `InfoType`: Information type enum
 //! - `CommandResult`: Result of parsing command line arguments

@@ -55,7 +56,7 @@ pub(super) const LONG_VERSION: &str = concat!(
 );

 /// Known subcommands. When the first arg matches one of these, it is treated as a subcommand.
-pub const KNOWN_SUBCOMMANDS: &[&str] = &["server", "info"];
+pub const KNOWN_SUBCOMMANDS: &[&str] = &["server", "info", "tls"];

 /// Preprocess argv for legacy compatibility: `rustfs <volume>` and `rustfs --address ...` are
 /// treated as `rustfs server <volume>` and `rustfs server --address ...` respectively.
@@ -102,6 +103,8 @@ pub enum Commands {
    Server(Box<ServerOpts>),
    /// Display system information
    Info(InfoOpts),
+    /// Inspect TLS certificate directory layout and parsing status
+    Tls(TlsOpts),
 }

 /// Information type to display
@@ -135,6 +138,28 @@ pub struct InfoOpts {
    pub info_type: Option<InfoType>,
 }

+/// TLS diagnostic subcommand options
+#[derive(Args, Clone)]
+pub struct TlsOpts {
+    #[command(subcommand)]
+    pub command: TlsCommands,
+}
+
+/// TLS diagnostic subcommands
+#[derive(Subcommand, Clone)]
+pub enum TlsCommands {
+    /// Inspect a TLS certificate directory
+    Inspect(TlsInspectOpts),
+}
+
+/// TLS inspect options
+#[derive(Args, Clone)]
+pub struct TlsInspectOpts {
+    /// TLS directory to inspect
+    #[arg(long = "path", alias = "tls-path", value_parser = NonEmptyStringValueParser::new())]
+    pub path: String,
+}
+
 /// Server subcommand options
 #[derive(Args, Clone)]
 pub struct ServerOpts {
@@ -263,6 +288,8 @@ pub enum CommandResult {
    Server(Box<super::Config>),
    /// Info command with options
    Info(InfoOpts),
+    /// TLS command with options
+    Tls(TlsOpts),
 }

 /// Create default ServerOpts from environment variables
@@ -24,7 +24,6 @@ use rustfs_config::{
 };
 use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, Masked};
 use std::collections::HashSet;
-use std::net::SocketAddr;
 use std::sync::{Mutex, OnceLock};

 pub(crate) const LEGACY_ENV_RUSTFS_ROOT_USER: &str = "RUSTFS_ROOT_USER";
@@ -179,10 +178,6 @@ impl Config {
        DEFAULT_ACCESS_KEY.eq(&self.access_key) && DEFAULT_SECRET_KEY.eq(&self.secret_key)
    }

-    pub fn default_credentials_allowed_for_addr(&self, server_addr: SocketAddr, allow_insecure_defaults: bool) -> bool {
-        !self.is_using_default_credentials() || server_addr.ip().is_loopback() || allow_insecure_defaults
-    }
-
    /// Create Config from Opt
    pub(super) fn from_opt(opt: Opt) -> std::io::Result<Self> {
        let Opt {
@@ -15,7 +15,7 @@
 #[cfg(test)]
 #[allow(unsafe_op_in_unsafe_fn)]
 mod tests {
-    use crate::config::{Config, Opt};
+    use crate::config::{CommandResult, Config, Opt, TlsCommands};
    use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION};
    use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY};
    use rustfs_ecstore::disks_layout::DisksLayout;
@@ -72,6 +72,34 @@ mod tests {
        assert_eq!(opt_legacy.console_address, opt_server.console_address);
    }

+    #[test]
+    #[serial]
+    fn test_tls_inspect_subcommand_parses() {
+        let result =
+            Opt::parse_command(["rustfs", "tls", "inspect", "--path", "/tmp/certs"]).expect("tls inspect command should parse");
+
+        match result {
+            CommandResult::Tls(opts) => match opts.command {
+                TlsCommands::Inspect(inspect) => assert_eq!(inspect.path, "/tmp/certs"),
+            },
+            _ => panic!("expected TLS command result"),
+        }
+    }
+
+    #[test]
+    #[serial]
+    fn test_tls_inspect_subcommand_parses_tls_path_alias() {
+        let result =
+            Opt::parse_command(["rustfs", "tls", "inspect", "--tls-path", "/tmp/certs"]).expect("tls inspect alias should parse");
+
+        match result {
+            CommandResult::Tls(opts) => match opts.command {
+                TlsCommands::Inspect(inspect) => assert_eq!(inspect.path, "/tmp/certs"),
+            },
+            _ => panic!("expected TLS command result"),
+        }
+    }
+
    #[test]
    #[serial]
    fn test_default_console_configuration() {
@@ -113,24 +141,6 @@ mod tests {
        assert_eq!(config.buffer_profile, "GeneralPurpose");
    }

-    #[test]
-    fn default_credentials_allowed_only_for_loopback_or_explicit_opt_in() {
-        let config = Config::new("0.0.0.0:9000", vec!["/tmp/rustfs-vol1".to_string()]);
-
-        assert!(!config.default_credentials_allowed_for_addr("0.0.0.0:9000".parse().unwrap(), false));
-        assert!(config.default_credentials_allowed_for_addr("127.0.0.1:9000".parse().unwrap(), false));
-        assert!(config.default_credentials_allowed_for_addr("0.0.0.0:9000".parse().unwrap(), true));
-    }
-
-    #[test]
-    fn custom_credentials_allowed_on_non_loopback() {
-        let mut config = Config::new("0.0.0.0:9000", vec!["/tmp/rustfs-vol1".to_string()]);
-        config.access_key = "custom-access-key".to_string();
-        config.secret_key = "custom-secret-key".to_string();
-
-        assert!(config.default_credentials_allowed_for_addr("0.0.0.0:9000".parse().unwrap(), false));
-    }
-
    #[test]
    #[serial]
    fn test_custom_console_configuration() {
@@ -51,6 +51,7 @@ mod config_test;

 // Re-export public types
 pub use cli::{CommandResult, InfoOpts, InfoType};
+pub use cli::{TlsCommands, TlsInspectOpts, TlsOpts};
 pub use config_struct::Config;
 pub use info::execute_info;
 pub use opt::Opt;
@@ -98,6 +98,9 @@ impl Opt {
                // This should not happen in parse_from, as it's handled by parse_command
                panic!("Info command should be handled by parse_command");
            }
+            Some(Commands::Tls(_)) => {
+                panic!("TLS command should be handled by parse_command");
+            }
            None => {
                // Default to server with empty volumes (will be filled from env)
                Self::from_server_opts(default_server_opts())
@@ -129,6 +132,7 @@ impl Opt {
        };
        match cli.command {
            Some(Commands::Info(opts)) => Ok(CommandResult::Info(opts)),
+            Some(Commands::Tls(opts)) => Ok(CommandResult::Tls(opts)),
            Some(Commands::Server(opts)) => Self::server_command_result(Self::from_server_opts(*opts)),
            None => {
                // Default to server with empty volumes (will be filled from env)
@@ -157,7 +161,7 @@ impl Opt {
        let cli = Cli::try_parse_from(args)?;
        match cli.command {
            Some(Commands::Server(opts)) => Ok(Self::from_server_opts(*opts)),
-            Some(Commands::Info(_)) => Err(clap::Error::new(clap::error::ErrorKind::DisplayHelp)),
+            Some(Commands::Info(_)) | Some(Commands::Tls(_)) => Err(clap::Error::new(clap::error::ErrorKind::DisplayHelp)),
            None => {
                // Default to server with empty volumes
                Ok(Self::from_server_opts(default_server_opts()))
@@ -55,7 +55,6 @@ use crate::server::{
 };
 use crate::startup_fs_guard::enforce_unsupported_fs_policy;
 use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
-use rustfs_config::ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS;
 use rustfs_credentials::init_global_action_credentials;
 use rustfs_ecstore::store::init_lock_clients;
 use rustfs_ecstore::{
@@ -77,7 +76,7 @@ use rustfs_ecstore::{
 };
 use rustfs_iam::init_iam_sys;
 use rustfs_obs::{init_obs, set_global_guard};
-use rustfs_utils::{get_env_bool, net::parse_and_resolve_address};
+use rustfs_utils::net::parse_and_resolve_address;
 use rustls::crypto::aws_lc_rs::default_provider;
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
 use std::path::{Path, PathBuf};
@@ -308,8 +307,7 @@ impl RustFSServerBuilder {
        // Trusted proxies.
        rustfs_trusted_proxies::init();

-        // Resolve listen address before credential initialization so unsafe
-        // default credentials can fail before the server binds a listener.
+        // Resolve listen address before endpoint/global initialization.
        let server_addr =
            parse_and_resolve_address(config.address.as_str()).map_err(|e| ServerError::Init(format!("address: {e}")))?;

@@ -322,14 +320,6 @@ impl RustFSServerBuilder {
            ));
        }

-        let allow_insecure_defaults = get_env_bool(ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS, false);
-        if !config.default_credentials_allowed_for_addr(server_addr, allow_insecure_defaults) {
-            return Err(ServerError::Init(
-                "default root credentials are not allowed on non-loopback listeners; set access_key and secret_key to non-default values, bind to loopback, or set RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS=true for local development only"
-                    .to_string(),
-            ));
-        }
-
        // Credentials.
        init_global_action_credentials(Some(config.access_key.clone()), Some(config.secret_key.clone()))
            .map_err(|e| ServerError::Init(format!("credentials: {e:?}")))?;
@@ -217,6 +217,7 @@ impl From<StorageError> for ApiError {
            StorageError::BucketExists(_) => S3ErrorCode::BucketAlreadyOwnedByYou,
            StorageError::StorageFull => S3ErrorCode::ServiceUnavailable,
            StorageError::SlowDown => S3ErrorCode::SlowDown,
+            StorageError::NamespaceLockQuorumUnavailable { .. } => S3ErrorCode::ServiceUnavailable,
            StorageError::DecommissionNotStarted => S3ErrorCode::InvalidRequest,
            StorageError::DecommissionAlreadyRunning => S3ErrorCode::InvalidRequest,
            StorageError::RebalanceAlreadyRunning => S3ErrorCode::InvalidRequest,
@@ -413,6 +414,16 @@ mod tests {
            (StorageError::BucketExists("test".into()), S3ErrorCode::BucketAlreadyOwnedByYou),
            (StorageError::StorageFull, S3ErrorCode::ServiceUnavailable),
            (StorageError::SlowDown, S3ErrorCode::SlowDown),
+            (
+                StorageError::NamespaceLockQuorumUnavailable {
+                    mode: "write",
+                    bucket: "bucket".into(),
+                    object: "object".into(),
+                    required: 3,
+                    achieved: 2,
+                },
+                S3ErrorCode::ServiceUnavailable,
+            ),
            (StorageError::DecommissionNotStarted, S3ErrorCode::InvalidRequest),
            (StorageError::DecommissionAlreadyRunning, S3ErrorCode::InvalidRequest),
            (StorageError::RebalanceAlreadyRunning, S3ErrorCode::InvalidRequest),
@@ -460,6 +471,23 @@ mod tests {
        assert!(s3_error.source().is_some());
    }

+    #[test]
+    fn test_namespace_lock_quorum_failure_maps_to_service_unavailable_status() {
+        let api_error: ApiError = StorageError::NamespaceLockQuorumUnavailable {
+            mode: "write",
+            bucket: "bucket".into(),
+            object: "object".into(),
+            required: 3,
+            achieved: 2,
+        }
+        .into();
+
+        let s3_error: S3Error = api_error.into();
+
+        assert_eq!(*s3_error.code(), S3ErrorCode::ServiceUnavailable);
+        assert_eq!(s3_error.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
+    }
+
    #[test]
    fn test_api_error_to_s3_error_without_source() {
        let api_error = ApiError {
@@ -69,6 +69,7 @@ pub mod protocols;
 pub mod server;
 pub mod startup_fs_guard;
 pub mod storage;
+pub mod tls;
 pub mod update;
 pub mod version;

@@ -36,7 +36,6 @@ use rustfs::server::{
 };
 use rustfs::startup_fs_guard::enforce_unsupported_fs_policy;
 use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
-use rustfs_config::ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS;
 use rustfs_credentials::init_global_action_credentials;
 use rustfs_ecstore::store::init_lock_clients;
 use rustfs_ecstore::{
@@ -62,7 +61,7 @@ use rustfs_iam::{init_iam_sys, init_oidc_sys};
 use rustfs_obs::{init_metrics_runtime, init_obs, set_global_guard};
 use rustfs_scanner::init_data_scanner;
 use rustfs_utils::{
-    ExternalEnvCompatReport, apply_external_env_compat, get_env_bool, get_env_bool_with_aliases, net::parse_and_resolve_address,
+    ExternalEnvCompatReport, apply_external_env_compat, get_env_bool_with_aliases, net::parse_and_resolve_address,
 };
 use rustls::crypto::aws_lc_rs::default_provider;
 use std::io::{Error, Result};
@@ -135,12 +134,7 @@ fn is_using_default_credentials(config: &rustfs::config::Config) -> bool {
    config.is_using_default_credentials()
 }

-const DEFAULT_CREDENTIALS_WARNING_MESSAGE: &str = "Detected default root credentials; set RUSTFS_ACCESS_KEY and RUSTFS_SECRET_KEY to non-default values, or use RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS=true only for local development";
-const DEFAULT_CREDENTIALS_ERROR_MESSAGE: &str = "Default root credentials are not allowed on non-loopback listeners; set RUSTFS_ACCESS_KEY and RUSTFS_SECRET_KEY to non-default values, bind to loopback, or set RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS=true for local development only";
-
-fn allow_insecure_default_credentials() -> bool {
-    get_env_bool(ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS, false)
-}
+const DEFAULT_CREDENTIALS_WARNING_MESSAGE: &str = "Detected default root credentials; set RUSTFS_ACCESS_KEY and RUSTFS_SECRET_KEY to non-default values for production deployments";

 async fn async_main() -> Result<()> {
    // Parse command line arguments
@@ -159,10 +153,15 @@ async fn async_main() -> Result<()> {
        return Ok(());
    }

+    if let rustfs::config::CommandResult::Tls(opts) = &command_result {
+        rustfs::tls::execute_tls(opts)?;
+        return Ok(());
+    }
+
    // Get config for server command
    let config = match command_result {
        rustfs::config::CommandResult::Server(cfg) => cfg,
-        rustfs::config::CommandResult::Info(_) => unreachable!(),
+        rustfs::config::CommandResult::Info(_) | rustfs::config::CommandResult::Tls(_) => unreachable!(),
    };

    // Initialize the global config snapshot for info command
@@ -270,11 +269,6 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
    let server_port = server_addr.port();
    let server_address = server_addr.to_string();

-    if !config.default_credentials_allowed_for_addr(server_addr, allow_insecure_default_credentials()) {
-        error!("{DEFAULT_CREDENTIALS_ERROR_MESSAGE}");
-        return Err(Error::other(DEFAULT_CREDENTIALS_ERROR_MESSAGE));
-    }
-
    if is_using_default_credentials(&config) {
        warn!("{}", DEFAULT_CREDENTIALS_WARNING_MESSAGE);
    }
@@ -821,12 +815,9 @@ mod tests {

    #[test]
    fn default_credentials_messages_are_actionable_without_exposing_values() {
-        for message in [DEFAULT_CREDENTIALS_WARNING_MESSAGE, DEFAULT_CREDENTIALS_ERROR_MESSAGE] {
-            assert!(message.contains(rustfs_config::ENV_RUSTFS_ACCESS_KEY));
-            assert!(message.contains(rustfs_config::ENV_RUSTFS_SECRET_KEY));
-            assert!(message.contains(ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS));
-            assert!(!message.contains(rustfs_credentials::DEFAULT_ACCESS_KEY));
-            assert!(!message.contains(rustfs_credentials::DEFAULT_SECRET_KEY));
-        }
+        assert!(DEFAULT_CREDENTIALS_WARNING_MESSAGE.contains(rustfs_config::ENV_RUSTFS_ACCESS_KEY));
+        assert!(DEFAULT_CREDENTIALS_WARNING_MESSAGE.contains(rustfs_config::ENV_RUSTFS_SECRET_KEY));
+        assert!(!DEFAULT_CREDENTIALS_WARNING_MESSAGE.contains(rustfs_credentials::DEFAULT_ACCESS_KEY));
+        assert!(!DEFAULT_CREDENTIALS_WARNING_MESSAGE.contains(rustfs_credentials::DEFAULT_SECRET_KEY));
    }
 }
@@ -380,7 +380,7 @@ impl PathCategory {
            PathCategory::AdminApi
        } else if path.starts_with("/rustfs/console") {
            PathCategory::Console
-        } else if path.starts_with("/minio/health/") {
+        } else if path == "/health" || path.starts_with("/health/") {
            PathCategory::Probe
        } else {
            PathCategory::S3DataPlane
@@ -725,8 +725,9 @@ mod tests {

    #[test]
    fn test_path_category_classify_probe() {
-        assert_eq!(PathCategory::classify("/minio/health/live"), PathCategory::Probe);
-        assert_eq!(PathCategory::classify("/minio/health/ready"), PathCategory::Probe);
+        assert_eq!(PathCategory::classify("/health"), PathCategory::Probe);
+        assert_eq!(PathCategory::classify("/health/live"), PathCategory::Probe);
+        assert_eq!(PathCategory::classify("/health/ready"), PathCategory::Probe);
    }

    #[test]
@@ -13,13 +13,13 @@
 // limitations under the License.

 use crate::admin::console::is_console_path;
-use crate::admin::handlers::health::{build_health_payload, collect_dependency_readiness, health_check_state, probe_from_path};
+use crate::admin::handlers::health::{HealthProbe, build_health_response_parts};
 use crate::error::ApiError;
 use crate::server::cors;
 use crate::server::hybrid::HybridBody;
 use crate::server::{
-    ADMIN_PREFIX, CONSOLE_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, MINIO_ADMIN_V3_PREFIX, RPC_PREFIX,
-    RUSTFS_ADMIN_PREFIX,
+    ADMIN_PREFIX, CONSOLE_PREFIX, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX,
+    MINIO_ADMIN_V3_PREFIX, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, active_http_requests, collect_dependency_readiness_report,
 };
 use crate::storage::apply_cors_headers;
 use crate::storage::request_context::{RequestContext, extract_request_id_from_headers};
@@ -625,10 +625,58 @@ fn health_endpoint_enabled() -> bool {
    rustfs_utils::get_env_bool(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, rustfs_config::DEFAULT_HEALTH_ENDPOINT_ENABLE)
 }

+fn health_compat_busy_check_enabled() -> bool {
+    rustfs_utils::get_env_bool(
+        rustfs_config::ENV_HEALTH_COMPAT_BUSY_CHECK_ENABLE,
+        rustfs_config::DEFAULT_HEALTH_COMPAT_BUSY_CHECK_ENABLE,
+    )
+}
+
+fn health_compat_busy_max_active_requests() -> u64 {
+    rustfs_utils::get_env_usize(
+        rustfs_config::ENV_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS,
+        rustfs_config::DEFAULT_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS,
+    ) as u64
+}
+
+fn health_compat_kms_ready_check_enabled() -> bool {
+    rustfs_utils::get_env_bool(
+        rustfs_config::ENV_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE,
+        rustfs_config::DEFAULT_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE,
+    )
+}
+
+fn resolve_public_health_probe(method: &Method, path: &str) -> Option<HealthProbe> {
+    if (method != Method::GET && method != Method::HEAD) || !health_endpoint_enabled() {
+        return None;
+    }
+
+    match path {
+        HEALTH_PREFIX | HEALTH_COMPAT_LIVE_PATH => Some(HealthProbe::Liveness),
+        HEALTH_READY_PATH => Some(HealthProbe::Readiness),
+        _ => None,
+    }
+}
+
+fn alias_busy_threshold_exceeded(active_requests: u64) -> bool {
+    if !health_compat_busy_check_enabled() {
+        return false;
+    }
+
+    let max_active_requests = health_compat_busy_max_active_requests();
+    max_active_requests > 0 && active_requests >= max_active_requests
+}
+
 fn is_public_health_endpoint_request(method: &Method, path: &str) -> bool {
-    (method == Method::GET || method == Method::HEAD)
-        && (path == HEALTH_PREFIX || path == HEALTH_READY_PATH)
-        && health_endpoint_enabled()
+    resolve_public_health_probe(method, path).is_some()
+}
+
+async fn health_kms_ready() -> bool {
+    let Some(service_manager) = rustfs_kms::get_global_kms_service_manager() else {
+        return true;
+    };
+
+    matches!(service_manager.get_status().await, rustfs_kms::KmsServiceStatus::Running)
 }

 async fn build_public_health_http_response<RestBody, GrpcBody>(
@@ -638,18 +686,41 @@ async fn build_public_health_http_response<RestBody, GrpcBody>(
 where
    RestBody: From<Bytes>,
 {
-    let probe = probe_from_path(&path);
-    let (storage_ready, iam_ready) = collect_dependency_readiness().await;
-    let health = health_check_state(storage_ready, iam_ready, probe);
-    let body = if method == Method::HEAD {
-        Bytes::new()
+    let probe = resolve_public_health_probe(&method, path.as_str())
+        .expect("public health endpoint request should always resolve health probe");
+
+    if probe == HealthProbe::Readiness && alias_busy_threshold_exceeded(active_http_requests()) {
+        let retry_after = HeaderValue::from_static("5");
+        let body_bytes = Bytes::from_static(b"{\"status\":\"busy\",\"ready\":false}");
+        let mut builder = Response::builder()
+            .status(StatusCode::TOO_MANY_REQUESTS)
+            .header(http::header::CONTENT_TYPE, "application/json")
+            .header(http::header::RETRY_AFTER, retry_after);
+        if let Ok(val) = HeaderValue::from_str(&body_bytes.len().to_string()) {
+            builder = builder.header(http::header::CONTENT_LENGTH, val);
+        }
+        return builder
+            .body(HybridBody::Rest {
+                rest_body: RestBody::from(body_bytes),
+            })
+            .expect("failed to build health busy response");
+    }
+
+    let readiness_report = collect_dependency_readiness_report().await;
+    let kms_ready = if probe == HealthProbe::Readiness && health_compat_kms_ready_check_enabled() {
+        Some(health_kms_ready().await)
    } else {
-        let payload = build_health_payload(health, storage_ready, iam_ready, "rustfs-endpoint", None);
-        Bytes::from(serde_json::to_vec(&payload).unwrap_or_else(|_| b"{}".to_vec()))
+        None
    };

+    let response_parts = build_health_response_parts(method, probe, &readiness_report, "rustfs-endpoint", None, kms_ready);
+    let body = response_parts
+        .payload
+        .map(|payload| Bytes::from(serde_json::to_vec(&payload).unwrap_or_else(|_| b"{}".to_vec())))
+        .unwrap_or_default();
+
    Response::builder()
-        .status(health.status_code)
+        .status(response_parts.status_code)
        .header(http::header::CONTENT_TYPE, "application/json")
        .body(HybridBody::Rest {
            rest_body: RestBody::from(body),
@@ -818,7 +889,8 @@ impl ConditionalCorsLayer {
    }

    /// Exact paths that should be excluded from being treated as S3 paths.
-    const EXCLUDED_EXACT_PATHS: &'static [&'static str] = &["/health", "/health/ready", "/profile/cpu", "/profile/memory"];
+    const EXCLUDED_EXACT_PATHS: &'static [&'static str] =
+        &["/health", "/health/live", "/health/ready", "/profile/cpu", "/profile/memory"];

    fn is_s3_path(path: &str) -> bool {
        // Exclude Admin, Console, RPC, and configured special paths
@@ -1207,6 +1279,73 @@ mod tests {
        .await;
    }

+    #[tokio::test]
+    #[serial]
+    async fn public_health_endpoint_layer_handles_health_live_path() {
+        async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"))], async {
+            let inner = CountingHybridService::default();
+            let calls = inner.calls();
+            let mut service = PublicHealthEndpointLayer.layer(inner);
+
+            let response = service
+                .call(
+                    Request::builder()
+                        .method(Method::GET)
+                        .uri(HEALTH_COMPAT_LIVE_PATH)
+                        .body(Full::<Bytes>::from(Bytes::new()))
+                        .expect("request"),
+                )
+                .await
+                .expect("health response");
+
+            assert_eq!(response.status(), StatusCode::OK);
+            assert_eq!(calls.load(Ordering::SeqCst), 0);
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn public_health_endpoint_layer_forwards_unknown_health_path_when_endpoint_disabled() {
+        async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("false"))], async {
+            let inner = CountingHybridService::default();
+            let calls = inner.calls();
+            let mut service = PublicHealthEndpointLayer.layer(inner);
+
+            let response = service
+                .call(
+                    Request::builder()
+                        .method(Method::GET)
+                        .uri("/health/live")
+                        .body(Full::<Bytes>::from(Bytes::new()))
+                        .expect("request"),
+                )
+                .await
+                .expect("inner response");
+
+            assert_eq!(response.status(), StatusCode::IM_A_TEAPOT);
+            assert_eq!(calls.load(Ordering::SeqCst), 1);
+        })
+        .await;
+    }
+
+    #[test]
+    fn alias_busy_threshold_exceeded_requires_switch_and_positive_threshold() {
+        with_var(rustfs_config::ENV_HEALTH_COMPAT_BUSY_CHECK_ENABLE, Some("true"), || {
+            with_var(rustfs_config::ENV_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS, Some("2"), || {
+                assert!(!alias_busy_threshold_exceeded(1));
+                assert!(alias_busy_threshold_exceeded(2));
+                assert!(alias_busy_threshold_exceeded(3));
+            });
+        });
+
+        with_var(rustfs_config::ENV_HEALTH_COMPAT_BUSY_CHECK_ENABLE, Some("false"), || {
+            with_var(rustfs_config::ENV_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS, Some("1"), || {
+                assert!(!alias_busy_threshold_exceeded(100));
+            });
+        });
+    }
+
    #[tokio::test]
    #[serial]
    async fn public_health_endpoint_layer_forwards_health_when_endpoint_disabled() {
@@ -47,12 +47,16 @@ pub(crate) use module_switch::{
    refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, validate_module_switch_update,
 };
 pub(crate) use prefix::{
-    ADMIN_PREFIX, CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, LICENSE, MINIO_ADMIN_PREFIX,
-    MINIO_ADMIN_V3_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, TONIC_PREFIX, VERSION,
+    ADMIN_PREFIX, CONSOLE_PREFIX, FAVICON_PATH, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, LICENSE,
+    MINIO_ADMIN_PREFIX, MINIO_ADMIN_V3_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX,
+    TONIC_PREFIX, VERSION,
 };
 pub(crate) use readiness::DependencyReadiness;
+pub(crate) use readiness::DependencyReadinessReport;
+pub(crate) use readiness::ReadinessDegradedReason;
 pub(crate) use readiness::ReadinessGateLayer;
 pub(crate) use readiness::collect_dependency_readiness;
+pub(crate) use readiness::collect_dependency_readiness_report;
 pub use readiness::publish_ready_when_runtime_ready;

 #[derive(Clone, Copy, Debug)]
@@ -31,6 +31,9 @@ pub(crate) const HEALTH_PREFIX: &str = "/health";
 /// This path is used to check dependency readiness and may return 503.
 pub(crate) const HEALTH_READY_PATH: &str = "/health/ready";

+/// Health liveness probe compatibility alias path.
+pub(crate) const HEALTH_COMPAT_LIVE_PATH: &str = "/health/live";
+
 /// Predefined administrative prefix for RustFS server routes.
 /// This prefix is used for endpoints that handle administrative tasks
 /// such as configuration, monitoring, and management.
@@ -18,7 +18,10 @@ use http::{Request as HttpRequest, Response, StatusCode};
 use http_body::Body;
 use http_body_util::{BodyExt, Full};
 use hyper::body::Incoming;
+use metrics::{counter, gauge};
 use rustfs_common::GlobalReadiness;
+use rustfs_ecstore::global::is_dist_erasure;
+use rustfs_ecstore::global::{get_global_endpoints_opt, get_global_lock_clients};
 use rustfs_ecstore::new_object_layer_fn;
 use rustfs_ecstore::store_api::StorageAPI;
 use rustfs_iam::get_global_iam_sys;
@@ -39,11 +42,47 @@ use tracing::{debug, info};

 pub const STARTUP_RUNTIME_READINESS_MAX_WAIT: Duration = Duration::from_secs(30);
 pub const STARTUP_RUNTIME_READINESS_POLL_INTERVAL: Duration = Duration::from_secs(1);
+const METRIC_RUNTIME_READINESS_READY: &str = "rustfs_runtime_readiness_ready";
+const METRIC_RUNTIME_READINESS_DEGRADED_TOTAL: &str = "rustfs_runtime_readiness_degraded_total";

 #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
 pub struct DependencyReadiness {
    pub storage_ready: bool,
    pub iam_ready: bool,
+    pub lock_quorum_ready: bool,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReadinessDegradedReason {
+    StorageQuorumUnavailable,
+    IamNotReady,
+    LockQuorumUnavailable,
+    KmsNotReady,
+    StorageAndIamUnavailable,
+    StorageAndLockUnavailable,
+    IamAndLockUnavailable,
+    StorageIamAndLockUnavailable,
+}
+
+impl ReadinessDegradedReason {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ReadinessDegradedReason::StorageQuorumUnavailable => "storage_quorum_unavailable",
+            ReadinessDegradedReason::IamNotReady => "iam_not_ready",
+            ReadinessDegradedReason::LockQuorumUnavailable => "lock_quorum_unavailable",
+            ReadinessDegradedReason::KmsNotReady => "kms_not_ready",
+            ReadinessDegradedReason::StorageAndIamUnavailable => "storage_and_iam_unavailable",
+            ReadinessDegradedReason::StorageAndLockUnavailable => "storage_and_lock_unavailable",
+            ReadinessDegradedReason::IamAndLockUnavailable => "iam_and_lock_unavailable",
+            ReadinessDegradedReason::StorageIamAndLockUnavailable => "storage_iam_and_lock_unavailable",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct DependencyReadinessReport {
+    pub readiness: DependencyReadiness,
+    pub degraded_reasons: Vec<ReadinessDegradedReason>,
 }

 /// ReadinessGateLayer ensures that the system components (IAM, Storage)
@@ -117,6 +156,7 @@ where
                crate::server::PROFILE_MEMORY_PATH
                    | crate::server::PROFILE_CPU_PATH
                    | crate::server::HEALTH_PREFIX
+                    | crate::server::HEALTH_COMPAT_LIVE_PATH
                    | crate::server::HEALTH_READY_PATH
                    | crate::server::FAVICON_PATH
            );
@@ -185,6 +225,20 @@ struct StorageReadinessCacheEntry {
    storage_ready: bool,
 }

+#[derive(Debug, Clone, Copy)]
+struct LockQuorumCacheEntry {
+    captured_at: Instant,
+    status: LockQuorumStatus,
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct LockQuorumStatus {
+    pub ready: bool,
+    pub connected_clients: usize,
+    pub total_clients: usize,
+    pub required_quorum: usize,
+}
+
 const DISK_STATE_OK: &str = "ok";
 const DISK_STATE_UNFORMATTED: &str = "unformatted";
 const RUNTIME_STATE_RETURNING: &str = "returning";
@@ -201,6 +255,11 @@ fn storage_readiness_cache() -> &'static Mutex<Option<StorageReadinessCacheEntry
    CACHE.get_or_init(|| Mutex::new(None))
 }

+fn lock_quorum_status_cache() -> &'static Mutex<Option<LockQuorumCacheEntry>> {
+    static CACHE: OnceLock<Mutex<Option<LockQuorumCacheEntry>>> = OnceLock::new();
+    CACHE.get_or_init(|| Mutex::new(None))
+}
+
 async fn load_cached_storage_readiness() -> Option<bool> {
    let ttl = health_readiness_cache_ttl();
    if ttl.is_zero() {
@@ -228,6 +287,33 @@ async fn update_storage_readiness_cache(storage_ready: bool) {
    });
 }

+async fn load_cached_lock_quorum_status() -> Option<LockQuorumStatus> {
+    let ttl = health_readiness_cache_ttl();
+    if ttl.is_zero() {
+        return None;
+    }
+
+    let cache = lock_quorum_status_cache().lock().await;
+    let entry = cache.as_ref()?;
+    if entry.captured_at.elapsed() <= ttl {
+        return Some(entry.status);
+    }
+
+    None
+}
+
+async fn update_lock_quorum_status_cache(status: LockQuorumStatus) {
+    if health_readiness_cache_ttl().is_zero() {
+        return;
+    }
+
+    let mut cache = lock_quorum_status_cache().lock().await;
+    *cache = Some(LockQuorumCacheEntry {
+        captured_at: Instant::now(),
+        status,
+    });
+}
+
 fn disk_is_online_for_readiness(disk: &Disk) -> bool {
    let state_is_acceptable = disk.state.eq_ignore_ascii_case(DISK_STATE_OK)
        || disk.state.eq_ignore_ascii_case(rustfs_madmin::ITEM_ONLINE)
@@ -320,8 +406,33 @@ fn storage_ready_from_runtime_state(info: &StorageInfo) -> bool {
    })
 }

+fn degraded_reasons(storage_ready: bool, iam_ready_raw: bool, lock_quorum_ready: bool) -> Vec<ReadinessDegradedReason> {
+    match (storage_ready, iam_ready_raw, lock_quorum_ready) {
+        (true, true, true) => Vec::new(),
+        (false, false, false) => vec![ReadinessDegradedReason::StorageIamAndLockUnavailable],
+        (false, false, true) => vec![ReadinessDegradedReason::StorageAndIamUnavailable],
+        (false, true, false) => vec![ReadinessDegradedReason::StorageAndLockUnavailable],
+        (true, false, false) => vec![ReadinessDegradedReason::IamAndLockUnavailable],
+        (false, true, true) => vec![ReadinessDegradedReason::StorageQuorumUnavailable],
+        (true, false, true) => vec![ReadinessDegradedReason::IamNotReady],
+        (true, true, false) => vec![ReadinessDegradedReason::LockQuorumUnavailable],
+    }
+}
+
+fn record_readiness_report(report: &DependencyReadinessReport) {
+    let ready = report.readiness.storage_ready && report.readiness.iam_ready && report.readiness.lock_quorum_ready;
+    gauge!(METRIC_RUNTIME_READINESS_READY).set(if ready { 1.0 } else { 0.0 });
+    for reason in &report.degraded_reasons {
+        counter!(METRIC_RUNTIME_READINESS_DEGRADED_TOTAL, "reason" => reason.as_str()).increment(1);
+    }
+}
+
 pub async fn collect_dependency_readiness() -> DependencyReadiness {
-    let iam_ready = get_global_iam_sys().is_some_and(|sys| sys.is_ready());
+    collect_dependency_readiness_report().await.readiness
+}
+
+pub async fn collect_dependency_readiness_report() -> DependencyReadinessReport {
+    let iam_ready_raw = get_global_iam_sys().is_some_and(|sys| sys.is_ready());
    let storage_ready = if let Some(cached) = load_cached_storage_readiness().await {
        cached
    } else {
@@ -329,21 +440,47 @@ pub async fn collect_dependency_readiness() -> DependencyReadiness {
        update_storage_readiness_cache(computed).await;
        computed
    };
+    let lock_quorum_status = collect_lock_quorum_status().await;

-    DependencyReadiness {
+    let readiness = DependencyReadiness {
        storage_ready,
-        iam_ready: iam_ready && storage_ready,
+        iam_ready: iam_ready_raw,
+        lock_quorum_ready: lock_quorum_status.ready,
+    };
+    let report = DependencyReadinessReport {
+        degraded_reasons: degraded_reasons(readiness.storage_ready, iam_ready_raw, readiness.lock_quorum_ready),
+        readiness,
+    };
+    record_readiness_report(&report);
+    report
+}
+
+async fn collect_lock_quorum_status() -> LockQuorumStatus {
+    if let Some(cached) = load_cached_lock_quorum_status().await {
+        cached
+    } else {
+        let computed = collect_lock_quorum_status_uncached().await;
+        update_lock_quorum_status_cache(computed).await;
+        computed
    }
 }

 async fn collect_dependency_readiness_uncached() -> DependencyReadiness {
-    let iam_ready = get_global_iam_sys().is_some_and(|sys| sys.is_ready());
+    let iam_ready_raw = get_global_iam_sys().is_some_and(|sys| sys.is_ready());
    let storage_ready = collect_storage_readiness_uncached().await;
+    let lock_quorum_status = collect_lock_quorum_status_uncached().await;

-    DependencyReadiness {
+    let readiness = DependencyReadiness {
        storage_ready,
-        iam_ready: iam_ready && storage_ready,
-    }
+        iam_ready: iam_ready_raw,
+        lock_quorum_ready: lock_quorum_status.ready,
+    };
+    let report = DependencyReadinessReport {
+        degraded_reasons: degraded_reasons(readiness.storage_ready, iam_ready_raw, readiness.lock_quorum_ready),
+        readiness,
+    };
+    record_readiness_report(&report);
+    report.readiness
 }

 async fn collect_storage_readiness_uncached() -> bool {
@@ -355,6 +492,115 @@ async fn collect_storage_readiness_uncached() -> bool {
    }
 }

+fn set_lock_quorum_status(
+    online_hosts: &HashSet<String>,
+    set_endpoints: &[rustfs_ecstore::disk::endpoint::Endpoint],
+) -> LockQuorumStatus {
+    let total_clients = set_endpoints
+        .iter()
+        .map(rustfs_ecstore::disk::endpoint::Endpoint::host_port)
+        .filter(|host| !host.is_empty())
+        .collect::<HashSet<_>>();
+    let total_clients_len = total_clients.len();
+    if total_clients_len == 0 {
+        return LockQuorumStatus::default();
+    }
+
+    let connected_clients = total_clients.iter().filter(|host| online_hosts.contains(*host)).count();
+    let required_quorum = if total_clients_len > 1 {
+        (total_clients_len / 2) + 1
+    } else {
+        1
+    };
+
+    LockQuorumStatus {
+        ready: connected_clients >= required_quorum,
+        connected_clients,
+        total_clients: total_clients_len,
+        required_quorum,
+    }
+}
+
+fn aggregate_lock_quorum_status(
+    pool_endpoints: &rustfs_ecstore::endpoints::EndpointServerPools,
+    online_hosts: &HashSet<String>,
+) -> LockQuorumStatus {
+    let mut connected_clients = 0usize;
+    let mut total_clients = 0usize;
+    let mut required_quorum = 0usize;
+
+    for pool in pool_endpoints.as_ref() {
+        for set_idx in 0..pool.set_count {
+            let set_endpoints = pool
+                .endpoints
+                .as_ref()
+                .iter()
+                .filter(|endpoint| endpoint.set_idx == set_idx as i32)
+                .cloned()
+                .collect::<Vec<_>>();
+
+            let status = set_lock_quorum_status(online_hosts, &set_endpoints);
+            if status.total_clients == 0 {
+                return LockQuorumStatus::default();
+            }
+
+            connected_clients += status.connected_clients;
+            total_clients += status.total_clients;
+            required_quorum += status.required_quorum;
+
+            if !status.ready {
+                return LockQuorumStatus {
+                    ready: false,
+                    connected_clients,
+                    total_clients,
+                    required_quorum,
+                };
+            }
+        }
+    }
+
+    if total_clients == 0 {
+        LockQuorumStatus::default()
+    } else {
+        LockQuorumStatus {
+            ready: true,
+            connected_clients,
+            total_clients,
+            required_quorum,
+        }
+    }
+}
+
+async fn collect_lock_quorum_status_uncached() -> LockQuorumStatus {
+    if !is_dist_erasure().await {
+        return LockQuorumStatus {
+            ready: true,
+            connected_clients: 1,
+            total_clients: 1,
+            required_quorum: 1,
+        };
+    }
+
+    let Some(pool_endpoints) = get_global_endpoints_opt() else {
+        return LockQuorumStatus::default();
+    };
+    let Some(lock_clients) = get_global_lock_clients() else {
+        return LockQuorumStatus::default();
+    };
+
+    let online_hosts = futures::future::join_all(lock_clients.iter().map(|(host, client)| {
+        let host = host.clone();
+        let client = client.clone();
+        async move { (host, client.is_online().await) }
+    }))
+    .await
+    .into_iter()
+    .filter_map(|(host, online)| online.then_some(host))
+    .collect::<HashSet<_>>();
+
+    aggregate_lock_quorum_status(&pool_endpoints, &online_hosts)
+}
+
 pub async fn wait_for_runtime_readiness_with<F, Fut, ReadyFn>(
    max_wait: Duration,
    poll_interval: Duration,
@@ -370,17 +616,18 @@ where

    loop {
        let readiness = load_readiness().await;
-        if readiness.storage_ready && readiness.iam_ready {
+        if readiness.storage_ready && readiness.iam_ready && readiness.lock_quorum_ready {
            on_ready(readiness);
            return Ok(());
        }

        if tokio::time::Instant::now() >= startup_deadline {
            let reason = format!(
-                "startup readiness timed out after {}s: storage_ready={}, iam_ready={}",
+                "startup readiness timed out after {}s: storage_ready={}, iam_ready={}, lock_quorum_ready={}",
                max_wait.as_secs(),
                readiness.storage_ready,
-                readiness.iam_ready
+                readiness.iam_ready,
+                readiness.lock_quorum_ready
            );
            return Err(std::io::Error::other(reason));
        }
@@ -389,6 +636,7 @@ where
            target: "rustfs::server::readiness",
            storage_ready = readiness.storage_ready,
            iam_ready = readiness.iam_ready,
+            lock_quorum_ready = readiness.lock_quorum_ready,
            "Runtime readiness has not reached write quorum yet; delaying ready state publication"
        );
        tokio::time::sleep(poll_interval).await;
@@ -420,6 +668,7 @@ mod tests {
                future::ready(DependencyReadiness {
                    storage_ready: false,
                    iam_ready: false,
+                    lock_quorum_ready: false,
                })
            },
            |_| {
@@ -435,6 +684,34 @@ mod tests {
        assert_eq!(state_manager.current_state(), ServiceState::Starting);
    }

+    #[tokio::test]
+    async fn wait_for_runtime_readiness_with_does_not_publish_ready_when_lock_quorum_is_not_reached() {
+        let readiness = GlobalReadiness::new();
+        let state_manager = ServiceStateManager::new();
+
+        let err = wait_for_runtime_readiness_with(
+            Duration::ZERO,
+            Duration::from_millis(1),
+            || {
+                future::ready(DependencyReadiness {
+                    storage_ready: true,
+                    iam_ready: true,
+                    lock_quorum_ready: false,
+                })
+            },
+            |_| {
+                readiness.mark_stage(rustfs_common::SystemStage::FullReady);
+                state_manager.update(ServiceState::Ready);
+            },
+        )
+        .await
+        .expect_err("lock quorum should block readiness publication");
+
+        assert!(err.to_string().contains("lock_quorum_ready=false"));
+        assert!(!readiness.is_ready());
+        assert_eq!(state_manager.current_state(), ServiceState::Starting);
+    }
+
    #[test]
    fn storage_ready_from_runtime_state_returns_false_when_all_disks_faulty() {
        let info = StorageInfo {
@@ -546,4 +823,154 @@ mod tests {
            "if any set fails write quorum, readiness must be false"
        );
    }
+
+    #[test]
+    fn aggregate_lock_quorum_status_requires_each_set_to_meet_quorum() {
+        use rustfs_ecstore::disk::endpoint::Endpoint;
+        use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
+
+        let endpoints = vec![
+            Endpoint {
+                url: url::Url::parse("http://node1:9000/data1").unwrap(),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 0,
+                disk_idx: 0,
+            },
+            Endpoint {
+                url: url::Url::parse("http://node2:9000/data2").unwrap(),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 0,
+                disk_idx: 1,
+            },
+            Endpoint {
+                url: url::Url::parse("http://node3:9000/data3").unwrap(),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 1,
+                disk_idx: 0,
+            },
+            Endpoint {
+                url: url::Url::parse("http://node4:9000/data4").unwrap(),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 1,
+                disk_idx: 1,
+            },
+        ];
+        let pools = EndpointServerPools::from(vec![PoolEndpoints {
+            legacy: false,
+            set_count: 2,
+            drives_per_set: 2,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: String::new(),
+            platform: String::new(),
+        }]);
+
+        let status = aggregate_lock_quorum_status(
+            &pools,
+            &["node1:9000", "node2:9000", "node3:9000", "node4:9000"]
+                .into_iter()
+                .map(str::to_string)
+                .collect::<HashSet<_>>(),
+        );
+
+        assert!(status.ready);
+        assert_eq!(status.connected_clients, 4);
+        assert_eq!(status.total_clients, 4);
+        assert_eq!(status.required_quorum, 4);
+    }
+
+    #[test]
+    fn aggregate_lock_quorum_status_fails_when_any_set_loses_quorum() {
+        use rustfs_ecstore::disk::endpoint::Endpoint;
+        use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
+
+        let endpoints = vec![
+            Endpoint {
+                url: url::Url::parse("http://node1:9000/data1").unwrap(),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 0,
+                disk_idx: 0,
+            },
+            Endpoint {
+                url: url::Url::parse("http://node2:9000/data2").unwrap(),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 0,
+                disk_idx: 1,
+            },
+            Endpoint {
+                url: url::Url::parse("http://node3:9000/data3").unwrap(),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 1,
+                disk_idx: 0,
+            },
+            Endpoint {
+                url: url::Url::parse("http://node4:9000/data4").unwrap(),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 1,
+                disk_idx: 1,
+            },
+        ];
+        let pools = EndpointServerPools::from(vec![PoolEndpoints {
+            legacy: false,
+            set_count: 2,
+            drives_per_set: 2,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: String::new(),
+            platform: String::new(),
+        }]);
+
+        let status =
+            aggregate_lock_quorum_status(&pools, &["node1:9000"].into_iter().map(str::to_string).collect::<HashSet<_>>());
+
+        assert!(!status.ready);
+    }
+
+    #[test]
+    fn degraded_reasons_include_lock_quorum_failures() {
+        assert_eq!(degraded_reasons(true, true, false), vec![ReadinessDegradedReason::LockQuorumUnavailable]);
+        assert_eq!(
+            degraded_reasons(false, true, false),
+            vec![ReadinessDegradedReason::StorageAndLockUnavailable]
+        );
+        assert_eq!(degraded_reasons(true, false, false), vec![ReadinessDegradedReason::IamAndLockUnavailable]);
+        assert_eq!(
+            degraded_reasons(false, false, false),
+            vec![ReadinessDegradedReason::StorageIamAndLockUnavailable]
+        );
+    }
+
+    #[tokio::test]
+    async fn lock_quorum_status_cache_roundtrip() {
+        let cache = lock_quorum_status_cache();
+        {
+            let mut guard = cache.lock().await;
+            *guard = None;
+        }
+
+        update_lock_quorum_status_cache(LockQuorumStatus {
+            ready: true,
+            connected_clients: 2,
+            total_clients: 3,
+            required_quorum: 2,
+        })
+        .await;
+
+        let cached = load_cached_lock_quorum_status().await;
+        assert_eq!(
+            cached,
+            Some(LockQuorumStatus {
+                ready: true,
+                connected_clients: 2,
+                total_clients: 3,
+                required_quorum: 2,
+            })
+        );
+    }
 }
@@ -107,11 +107,15 @@ impl Node for NodeService {
        debug!("PING");

        let ping_req = request.into_inner();
-        let ping_body = flatbuffers::root::<PingBody>(&ping_req.body);
-        if let Err(e) = ping_body {
-            error!("{}", e);
+        if ping_req.body.is_empty() {
+            debug!("ping_req received empty body; treating request as liveness probe");
        } else {
-            info!("ping_req:body(flatbuffer): {:?}", ping_body);
+            let ping_body = flatbuffers::root::<PingBody>(&ping_req.body);
+            if let Err(e) = ping_body {
+                warn!("invalid ping request body: {}", e);
+            } else {
+                info!("ping_req:body(flatbuffer): {:?}", ping_body);
+            }
        }

        let mut fbb = flatbuffers::FlatBufferBuilder::new();
@@ -992,6 +996,23 @@ mod tests {
        assert!(!ping_response.body.is_empty());
    }

+    #[tokio::test]
+    async fn test_ping_with_empty_body() {
+        let service = create_test_node_service();
+
+        let request = Request::new(PingRequest {
+            version: 1,
+            body: Bytes::new(),
+        });
+
+        let response = service.ping(request).await;
+        assert!(response.is_ok());
+
+        let ping_response = response.unwrap().into_inner();
+        assert_eq!(ping_response.version, 1);
+        assert!(!ping_response.body.is_empty());
+    }
+
    #[tokio::test]
    async fn test_heal_bucket_invalid_options() {
        let service = create_test_node_service();
@@ -19,6 +19,7 @@ use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, Multip
 use s3s::{S3Error, S3ErrorCode};

 const MAX_MULTIPART_UPLOADS_LIST: i32 = 1000;
+const MAX_MULTIPART_PART_NUMBER: i32 = 10000;

 #[derive(Debug, PartialEq, Eq)]
 pub(crate) struct ListPartsParams {
@@ -103,6 +104,17 @@ pub(crate) fn parse_list_parts_params(
    })
 }

+pub(crate) fn parse_upload_part_number(part_number: i32) -> Result<usize, S3Error> {
+    if !(1..=MAX_MULTIPART_PART_NUMBER).contains(&part_number) {
+        return Err(S3Error::with_message(
+            S3ErrorCode::InvalidArgument,
+            format!("partNumber must be between 1 and {MAX_MULTIPART_PART_NUMBER}"),
+        ));
+    }
+
+    Ok(part_number as usize)
+}
+
 pub(crate) fn parse_list_multipart_uploads_params(
    prefix: Option<String>,
    key_marker: Option<String>,
@@ -176,7 +188,7 @@ pub(crate) fn build_list_multipart_uploads_output(
 mod tests {
    use super::{
        MAX_MULTIPART_UPLOADS_LIST, build_list_multipart_uploads_output, build_list_parts_output,
-        parse_list_multipart_uploads_params, parse_list_parts_params,
+        parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
    };
    use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
    use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
@@ -319,6 +331,21 @@ mod tests {
        assert_eq!(err.message(), Some("part-number-marker must be non-negative"));
    }

+    #[test]
+    fn test_parse_upload_part_number_accepts_s3_range() {
+        assert_eq!(parse_upload_part_number(1).expect("part 1 should be valid"), 1);
+        assert_eq!(parse_upload_part_number(10000).expect("part 10000 should be valid"), 10000);
+    }
+
+    #[test]
+    fn test_parse_upload_part_number_rejects_out_of_s3_range() {
+        for part_number in [-1, 0, 10001] {
+            let err = parse_upload_part_number(part_number).expect_err("expected invalid part number");
+            assert_eq!(*err.code(), S3ErrorCode::InvalidArgument);
+            assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
+        }
+    }
+
    #[test]
    fn test_parse_list_multipart_uploads_params_defaults_and_valid_values() {
        let parsed =
@@ -0,0 +1,135 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::config::{TlsCommands, TlsInspectOpts, TlsOpts};
+use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY};
+use rustfs_tls_runtime::{CertDirectoryLoadOptions, TlsCertPairInspection, TlsCertPairStatus, TlsSource, inspect_cert_directory};
+use std::io::{Error, Result};
+
+pub fn execute_tls(opts: &TlsOpts) -> Result<()> {
+    match &opts.command {
+        TlsCommands::Inspect(inspect) => execute_tls_inspect(inspect),
+    }
+}
+
+fn execute_tls_inspect(opts: &TlsInspectOpts) -> Result<()> {
+    let tls_source = TlsSource::from_directory(opts.path.trim());
+    let tls_dir = tls_source.validate_directory().map_err(Error::other)?;
+    let inspection = inspect_cert_directory(CertDirectoryLoadOptions::builder(tls_dir, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY).build())?;
+
+    println!("TLS directory inspection");
+    println!("path: {}", inspection.directory.display());
+    if let Some(canonical_directory) = &inspection.canonical_directory {
+        println!("canonical path: {}", canonical_directory.display());
+    }
+    println!("root pair: {}", format_pair_summary(&inspection.root_pair));
+
+    let valid_domains = inspection.valid_domain_names();
+    let runtime_mode = if !valid_domains.is_empty() {
+        "multi-cert/SNI"
+    } else if inspection.has_valid_root_pair() {
+        "single-cert/default"
+    } else {
+        "no-usable-server-cert"
+    };
+    println!("runtime mode: {}", runtime_mode);
+    println!("valid domain pairs: {}", valid_domains.len());
+    if !valid_domains.is_empty() {
+        println!("valid domains: {}", valid_domains.join(", "));
+    }
+
+    if !inspection.skipped_directory_names.is_empty() {
+        println!("skipped internal directories: {}", inspection.skipped_directory_names.join(", "));
+    }
+
+    if inspection.domain_pairs.is_empty() {
+        println!("domain pairs: none");
+    } else {
+        println!("domain pairs:");
+        for domain in &inspection.domain_pairs {
+            println!("  - {}: {}", domain.domain_name, format_pair_summary(&domain.pair));
+        }
+    }
+
+    Ok(())
+}
+
+fn format_pair_summary(pair: &TlsCertPairInspection) -> String {
+    match &pair.status {
+        TlsCertPairStatus::MissingBoth => format!(
+            "missing cert/key (expected cert={}, key={})",
+            pair.cert_path.display(),
+            pair.key_path.display()
+        ),
+        TlsCertPairStatus::MissingCert => format!(
+            "missing cert (expected cert={}, key={})",
+            pair.cert_path.display(),
+            pair.key_path.display()
+        ),
+        TlsCertPairStatus::MissingKey => format!(
+            "missing key (expected cert={}, key={})",
+            pair.cert_path.display(),
+            pair.key_path.display()
+        ),
+        TlsCertPairStatus::Valid => {
+            format!("valid cert/key pair (cert={}, key={})", pair.cert_path.display(), pair.key_path.display())
+        }
+        TlsCertPairStatus::Invalid { error } => format!(
+            "invalid cert/key pair (cert={}, key={}): {}",
+            pair.cert_path.display(),
+            pair.key_path.display(),
+            error
+        ),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+    use tempfile::TempDir;
+
+    fn write_test_cert_pair(dir: &std::path::Path) {
+        let rcgen::CertifiedKey { cert, signing_key } =
+            rcgen::generate_simple_self_signed(vec!["example.com".to_string()]).expect("cert should generate");
+        fs::write(dir.join(RUSTFS_TLS_CERT), cert.pem()).expect("cert should write");
+        fs::write(dir.join(RUSTFS_TLS_KEY), signing_key.serialize_pem()).expect("key should write");
+    }
+
+    #[test]
+    fn execute_tls_inspect_accepts_valid_directory() {
+        let temp_dir = TempDir::new().expect("tempdir should create");
+        write_test_cert_pair(temp_dir.path());
+
+        let opts = TlsInspectOpts {
+            path: temp_dir.path().display().to_string(),
+        };
+
+        execute_tls_inspect(&opts).expect("tls inspect should succeed");
+    }
+
+    #[test]
+    fn format_pair_summary_reports_missing_key() {
+        let pair = TlsCertPairInspection {
+            cert_path: "/tmp/rustfs_cert.pem".into(),
+            key_path: "/tmp/rustfs_key.pem".into(),
+            status: TlsCertPairStatus::MissingKey,
+        };
+
+        let summary = format_pair_summary(&pair);
+        assert!(summary.contains("missing key"));
+        assert!(summary.contains("/tmp/rustfs_cert.pem"));
+        assert!(summary.contains("/tmp/rustfs_key.pem"));
+    }
+}
@@ -1,5 +1,5 @@
-RUSTFS_ACCESS_KEY=rustfs-dev-admin
-RUSTFS_SECRET_KEY=rustfs-dev-secret
+RUSTFS_ACCESS_KEY=rustfsadmin
+RUSTFS_SECRET_KEY=rustfsadmin

 RUSTFS_VOLUMES="http://node{1...4}:7000/data/rustfs{0...3}     http://node{5...8}:7000/data/rustfs{0...3}"
 RUSTFS_ADDRESS=":7000"
@@ -20,16 +20,16 @@ VOLUME=$2
 chmod +x $BIN
 mkdir -p $VOLUME

+export RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin}"
+export RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}"
 export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug"
 export RUST_BACKTRACE=full
-export RUSTFS_ACCESS_KEY=rustfs-e2e-admin
-export RUSTFS_SECRET_KEY=rustfs-e2e-secret
 $BIN $VOLUME > /tmp/rustfs.log 2>&1 &

 sleep 10

-export AWS_ACCESS_KEY_ID=$RUSTFS_ACCESS_KEY
-export AWS_SECRET_ACCESS_KEY=$RUSTFS_SECRET_KEY
+export AWS_ACCESS_KEY_ID="${RUSTFS_ACCESS_KEY:-rustfsadmin}"
+export AWS_SECRET_ACCESS_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}"
 export AWS_REGION=us-east-1
 export AWS_ENDPOINT_URL=http://localhost:9000
 export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug"
@@ -134,7 +134,7 @@ start_rustfs() {
    
    # Start RustFS in background with environment variables
    cd "$TARGET_DIR"
-    RUSTFS_ACCESS_KEY=rustfs-e2e-admin RUSTFS_SECRET_KEY=rustfs-e2e-secret \
+    RUSTFS_ACCESS_KEY=rustfsadmin RUSTFS_SECRET_KEY=rustfsadmin \
    RUSTFS_OBS_LOG_DIRECTORY="$TARGET_DIR/logs" \
    ./rustfs --address :9000 "$DATA_DIR" > rustfs.log 2>&1 &
    RUSTFS_PID=$!
@@ -15,8 +15,8 @@ PRECHECK_AUTO_CLEANUP="${PRECHECK_AUTO_CLEANUP:-true}"
 WAIT_PROBE_MODE="${WAIT_PROBE_MODE:-service}"
 COMPOSE_UP_NO_BUILD="${COMPOSE_UP_NO_BUILD:-false}"

-RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}"
-RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}"
+RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin}"
+RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}"
 RUSTFS_DOCKER_PLATFORM="${RUSTFS_DOCKER_PLATFORM:-}"
 RUSTFS_OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}"
 RUSTFS_UNSAFE_BYPASS_DISK_CHECK="${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}"
@@ -30,6 +30,7 @@ FAILOVER_INTERVAL_SECS="${FAILOVER_INTERVAL_SECS:-1}"
 BENCH_WAIT_MODE="${BENCH_WAIT_MODE:-ready}"

 BENCH_ENDPOINT="${BENCH_ENDPOINT:-http://127.0.0.1:9000}"
+BENCH_WARP_HOSTS="${BENCH_WARP_HOSTS:-http://127.0.0.1:9000,http://127.0.0.1:9001,http://127.0.0.1:9002,http://127.0.0.1:9003}"
 BENCH_BUCKET="${BENCH_BUCKET:-rustfs-four-node-bench}"
 BENCH_AUTO_NEW_BUCKET="${BENCH_AUTO_NEW_BUCKET:-true}"
 BENCH_BUCKET_PREFIX="${BENCH_BUCKET_PREFIX:-rustfs-four-node-bench}"
@@ -58,6 +59,7 @@ Options:
  --failover-node <nodeN>       node to stop during failover test (default: node4)
  --obs-endpoint <url>          RUSTFS_OBS_ENDPOINT (default: auto-select by mode)
  --bench-endpoint <url>        benchmark endpoint (default: http://127.0.0.1:9000)
+  --bench-warp-hosts <hosts>    comma-separated warp hosts (default: node1..node4)
  --bench-sizes <sizes>         comma list (default: 1KiB,4KiB,11Mi)
  --bench-concurrency <n>       benchmark concurrency
  --bench-concurrencies <list>  benchmark concurrency list (default: 8,16,32,64,128)
@@ -75,6 +77,7 @@ Environment:
  WAIT_PROBE_MODE (service|ready, default: service)
  WAIT_TIMEOUT_SECS FAILOVER_NODE FAILOVER_WARMUP_SECS FAILOVER_SAMPLE_SECS
  FAILOVER_INTERVAL_SECS BENCH_ENDPOINT BENCH_BUCKET BENCH_CONCURRENCY
+  BENCH_WARP_HOSTS (default: http://127.0.0.1:9000,http://127.0.0.1:9001,http://127.0.0.1:9002,http://127.0.0.1:9003)
  BENCH_CONCURRENCIES BENCH_DURATION BENCH_SIZES OUT_DIR
  BENCH_WAIT_MODE (ready|service, default: ready)
  BENCH_READY_TIMEOUT_SECS (default: 180)
@@ -585,7 +588,7 @@ run_benchmark() {
      cd "${PROJECT_ROOT}"
      ./scripts/run_object_batch_bench.sh \
        --tool warp \
-        --endpoint "${BENCH_ENDPOINT}" \
+        --endpoint "${BENCH_WARP_HOSTS}" \
        --access-key "${RUSTFS_ACCESS_KEY}" \
        --secret-key "${RUSTFS_SECRET_KEY}" \
        --bucket "${bench_bucket}" \
@@ -664,6 +667,10 @@ parse_args() {
        BENCH_ENDPOINT="$2"
        shift 2
        ;;
+      --bench-warp-hosts)
+        BENCH_WARP_HOSTS="$2"
+        shift 2
+        ;;
      --bench-sizes)
        BENCH_SIZES="$2"
        shift 2
@@ -108,12 +108,28 @@ print_dry_run_command() {

 normalize_warp_host() {
  local raw="$1"
-  raw="${raw#http://}"
-  raw="${raw#https://}"
-  raw="${raw%%/*}"
-  raw="${raw%%\?*}"
-  raw="${raw%%\#*}"
-  echo "$raw"
+  local item normalized
+  local -a parts
+  local -a hosts=()
+
+  IFS=',' read -r -a parts <<< "$raw"
+  for item in "${parts[@]}"; do
+    item="$(echo "$item" | xargs)"
+    [[ -z "$item" ]] && continue
+    item="${item#http://}"
+    item="${item#https://}"
+    item="${item%%/*}"
+    item="${item%%\?*}"
+    item="${item%%\#*}"
+    [[ -n "$item" ]] && hosts+=("$item")
+  done
+
+  if [[ ${#hosts[@]} -eq 0 ]]; then
+    return 0
+  fi
+
+  normalized="$(IFS=','; echo "${hosts[*]}")"
+  echo "$normalized"
 }

 parse_args() {
@@ -130,8 +130,8 @@ DEPLOY_MODE=existing S3_HOST=192.168.1.100 S3_PORT=9000 ./scripts/s3-tests/run.s

 ### Service Configuration

- `S3_ACCESS_KEY`: Main user access key (default: `rustfs-ci-admin`)
- `S3_SECRET_KEY`: Main user secret key (default: `rustfs-ci-secret`)
+- `S3_ACCESS_KEY`: Main user access key (default: `rustfsadmin`)
+- `S3_SECRET_KEY`: Main user secret key (default: `rustfsadmin`)
 - `S3_ALT_ACCESS_KEY`: Alt user access key (default: `rustfsalt`)
 - `S3_ALT_SECRET_KEY`: Alt user secret key (default: `rustfsalt`)
 - `S3_REGION`: S3 region (default: `us-east-1`)
@@ -420,8 +420,8 @@ curl http://192.168.1.100:9000/health

 # Verify S3 API is responding
 awscurl --service s3 --region us-east-1 \
-  --access_key rustfs-ci-admin \
-  --secret_key rustfs-ci-secret \
+  --access_key rustfsadmin \
+  --secret_key rustfsadmin \
  -X GET "http://192.168.1.100:9000/"
 ```

@@ -25,8 +25,8 @@ PYTHON_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.v
 export PATH="$HOME/Library/Python/${PYTHON_VERSION}/bin:$HOME/.local/bin:$PATH"

 # Configuration
-S3_ACCESS_KEY="${S3_ACCESS_KEY:-rustfs-ci-admin}"
-S3_SECRET_KEY="${S3_SECRET_KEY:-rustfs-ci-secret}"
+S3_ACCESS_KEY="${S3_ACCESS_KEY:-rustfsadmin}"
+S3_SECRET_KEY="${S3_SECRET_KEY:-rustfsadmin}"
 S3_ALT_ACCESS_KEY="${S3_ALT_ACCESS_KEY:-rustfsalt}"
 S3_ALT_SECRET_KEY="${S3_ALT_SECRET_KEY:-rustfsalt}"
 S3_REGION="${S3_REGION:-us-east-1}"
@@ -218,8 +218,8 @@ Environment Variables:
  RUSTFS_BINARY          - Path to RustFS binary (for binary mode, default: ./target/release/rustfs)
  S3_HOST                - S3 service host (default: 127.0.0.1)
  S3_PORT                - S3 service port (default: 9000)
-  S3_ACCESS_KEY          - Main user access key (default: rustfs-ci-admin)
-  S3_SECRET_KEY          - Main user secret key (default: rustfs-ci-secret)
+  S3_ACCESS_KEY          - Main user access key (default: rustfsadmin)
+  S3_SECRET_KEY          - Main user secret key (default: rustfsadmin)
  S3_ALT_ACCESS_KEY      - Alt user access key (default: rustfsalt)
  S3_ALT_SECRET_KEY      - Alt user secret key (default: rustfsalt)
  MAXFAIL                - Stop after N failures (default: 1)
@@ -690,7 +690,7 @@ envsubst < "${TEMPLATE_PATH}" > "${CONF_OUTPUT_PATH}" || {
 }

 # Step 7: Provision s3-tests alt user
-# Note: The configured main user is a system user and doesn't need to be created via API
+# Note: Main user (rustfsadmin) is a system user and doesn't need to be created via API
 log_info "Provisioning s3-tests alt user..."

 # Helper function to install Python packages with fallback for externally-managed environments
@@ -0,0 +1,466 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+CLUSTER_COMPOSE="${CLUSTER_COMPOSE:-${PROJECT_ROOT}/.docker/compose/docker-compose.cluster.local-build.yml}"
+PROJECT_NAME="${PROJECT_NAME:-rustfs-issue3031}"
+RUSTFS_IMAGE="${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}"
+BUILD_LOCAL_IMAGE="${BUILD_LOCAL_IMAGE:-true}"
+FORCE_BUILD="${FORCE_BUILD:-false}"
+KEEP_UP="${KEEP_UP:-false}"
+PRECHECK_AUTO_CLEANUP="${PRECHECK_AUTO_CLEANUP:-true}"
+WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}"
+S3_READY_TIMEOUT_SECS="${S3_READY_TIMEOUT_SECS:-120}"
+
+RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}"
+RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}"
+RUSTFS_UNSAFE_BYPASS_DISK_CHECK="${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}"
+RUSTFS_ISSUE3031_DIAG_ENABLE="${RUSTFS_ISSUE3031_DIAG_ENABLE:-true}"
+RUSTFS_OBS_LOG_STDOUT_ENABLED="${RUSTFS_OBS_LOG_STDOUT_ENABLED:-true}"
+RUSTFS_OBS_USE_STDOUT="${RUSTFS_OBS_USE_STDOUT:-false}"
+RUSTFS_OBS_LOGGER_LEVEL="${RUSTFS_OBS_LOGGER_LEVEL:-info}"
+RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT="${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}"
+RUSTFS_LOCK_ACQUIRE_TIMEOUT="${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}"
+
+ENDPOINT="${ENDPOINT:-http://127.0.0.1:9000}"
+WARP_HOST="${WARP_HOST:-node1:9000}"
+BUCKET="${BUCKET:-rustfs-multipart-repro}"
+SMOKE_OBJECTS="${SMOKE_OBJECTS:-16}"
+SMOKE_OBJECT_SIZE_MB="${SMOKE_OBJECT_SIZE_MB:-32}"
+SMOKE_PARALLELISM="${SMOKE_PARALLELISM:-8}"
+
+WARP_DURATION="${WARP_DURATION:-5m}"
+WARP_CONCURRENT="${WARP_CONCURRENT:-16}"
+WARP_PARTS="${WARP_PARTS:-16}"
+WARP_PART_SIZE="${WARP_PART_SIZE:-16MiB}"
+WARP_PART_CONCURRENT="${WARP_PART_CONCURRENT:-4}"
+
+MC_IMAGE="${MC_IMAGE:-minio/mc:latest}"
+WARP_IMAGE="${WARP_IMAGE:-minio/warp:latest}"
+OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/issue3031/$(date +%Y%m%d-%H%M%S)}"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/validate_issue_3031_docker.sh [options]
+
+Options:
+  --skip-build                  skip local Docker image build
+  --force-build                 rebuild even if the local image already exists
+  --keep-up                     keep cluster running after the script exits
+  --project-name <name>         docker compose project name
+  --out-dir <path>              output directory
+  --warp-duration <dur>         warp duration (default: 5m)
+  --warp-concurrent <n>         warp --concurrent (default: 16)
+  --warp-parts <n>              warp --parts (default: 16)
+  --warp-part-size <size>       warp --part.size (default: 16MiB)
+  --warp-part-concurrent <n>    warp --part.concurrent (default: 4)
+  --smoke-objects <n>           plain write smoke object count (default: 16)
+  --smoke-object-size-mb <n>    plain write smoke object size MiB (default: 32)
+  --smoke-parallelism <n>       plain write smoke parallelism (default: 8)
+  -h, --help                    show help
+
+Environment:
+  CLUSTER_COMPOSE PROJECT_NAME RUSTFS_IMAGE BUILD_LOCAL_IMAGE FORCE_BUILD KEEP_UP
+  RUSTFS_ACCESS_KEY RUSTFS_SECRET_KEY RUSTFS_UNSAFE_BYPASS_DISK_CHECK
+  RUSTFS_ISSUE3031_DIAG_ENABLE RUSTFS_OBS_LOG_STDOUT_ENABLED
+  RUSTFS_OBS_USE_STDOUT RUSTFS_OBS_LOGGER_LEVEL
+  RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT
+  RUSTFS_LOCK_ACQUIRE_TIMEOUT
+  S3_READY_TIMEOUT_SECS
+  ENDPOINT BUCKET
+  WARP_HOST
+  SMOKE_OBJECTS SMOKE_OBJECT_SIZE_MB SMOKE_PARALLELISM
+  WARP_DURATION WARP_CONCURRENT WARP_PARTS WARP_PART_SIZE WARP_PART_CONCURRENT
+  MC_IMAGE WARP_IMAGE OUT_DIR
+USAGE
+}
+
+log_info() {
+  printf '[INFO] %s\n' "$*"
+}
+
+log_warn() {
+  printf '[WARN] %s\n' "$*"
+}
+
+log_error() {
+  printf '[ERROR] %s\n' "$*" >&2
+}
+
+require_cmd() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    log_error "command not found: $1"
+    exit 1
+  fi
+}
+
+compose() {
+  docker compose \
+    --project-name "${PROJECT_NAME}" \
+    -f "${CLUSTER_COMPOSE}" \
+    "$@"
+}
+
+cleanup() {
+  if [[ "${KEEP_UP}" == "true" ]]; then
+    log_info "KEEP_UP=true, leaving cluster running"
+    return
+  fi
+
+  log_info "Stopping docker compose services"
+  compose down -v >/dev/null 2>&1 || true
+}
+
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --skip-build)
+        BUILD_LOCAL_IMAGE=false
+        shift
+        ;;
+      --keep-up)
+        KEEP_UP=true
+        shift
+        ;;
+      --force-build)
+        FORCE_BUILD=true
+        shift
+        ;;
+      --project-name)
+        PROJECT_NAME="$2"
+        shift 2
+        ;;
+      --out-dir)
+        OUT_DIR="$2"
+        shift 2
+        ;;
+      --warp-duration)
+        WARP_DURATION="$2"
+        shift 2
+        ;;
+      --warp-concurrent)
+        WARP_CONCURRENT="$2"
+        shift 2
+        ;;
+      --warp-parts)
+        WARP_PARTS="$2"
+        shift 2
+        ;;
+      --warp-part-size)
+        WARP_PART_SIZE="$2"
+        shift 2
+        ;;
+      --warp-part-concurrent)
+        WARP_PART_CONCURRENT="$2"
+        shift 2
+        ;;
+      --smoke-objects)
+        SMOKE_OBJECTS="$2"
+        shift 2
+        ;;
+      --smoke-object-size-mb)
+        SMOKE_OBJECT_SIZE_MB="$2"
+        shift 2
+        ;;
+      --smoke-parallelism)
+        SMOKE_PARALLELISM="$2"
+        shift 2
+        ;;
+      -h|--help)
+        usage
+        exit 0
+        ;;
+      *)
+        log_error "unknown argument: $1"
+        usage
+        exit 1
+        ;;
+    esac
+  done
+}
+
+wait_http_ok() {
+  local url="$1"
+  local start now
+  start="$(date +%s)"
+
+  while true; do
+    if curl -fsS --connect-timeout 2 --max-time 3 "${url}" >/dev/null 2>&1; then
+      return 0
+    fi
+
+    now="$(date +%s)"
+    if (( now - start >= WAIT_TIMEOUT_SECS )); then
+      log_error "timed out waiting for ${url}"
+      return 1
+    fi
+
+    sleep 2
+  done
+}
+
+cleanup_existing_project_containers() {
+  local existing_ids
+  existing_ids="$(docker ps -aq --filter "label=com.docker.compose.project=${PROJECT_NAME}")"
+
+  if [[ -z "${existing_ids}" ]]; then
+    return 0
+  fi
+
+  log_warn "Found existing containers for project ${PROJECT_NAME}."
+  docker ps -a --filter "label=com.docker.compose.project=${PROJECT_NAME}" --format '  - {{.Names}} ({{.Status}})'
+
+  if [[ "${PRECHECK_AUTO_CLEANUP}" == "true" ]]; then
+    log_info "PRECHECK_AUTO_CLEANUP=true, removing existing project containers."
+    # shellcheck disable=SC2086
+    docker rm -f ${existing_ids} >/dev/null
+  else
+    log_error "existing project containers detected and PRECHECK_AUTO_CLEANUP=false"
+    exit 1
+  fi
+}
+
+build_image_if_needed() {
+  if [[ "${BUILD_LOCAL_IMAGE}" != "true" ]]; then
+    log_info "Skipping local image build"
+    return
+  fi
+
+  if [[ "${FORCE_BUILD}" != "true" ]] && docker image inspect "${RUSTFS_IMAGE}" >/dev/null 2>&1; then
+    log_info "Reusing existing local image ${RUSTFS_IMAGE}; pass --force-build to rebuild"
+    return
+  fi
+
+  log_info "Building ${RUSTFS_IMAGE} from Dockerfile.source"
+  docker build -f "${PROJECT_ROOT}/Dockerfile.source" -t "${RUSTFS_IMAGE}" "${PROJECT_ROOT}"
+}
+
+start_cluster() {
+  mkdir -p "${OUT_DIR}"
+  cleanup_existing_project_containers
+
+  log_info "Starting 4-node cluster from ${CLUSTER_COMPOSE}"
+  compose up -d
+
+  log_info "Waiting for cluster health endpoint"
+  wait_http_ok "${ENDPOINT}/health"
+
+  log_info "Waiting for cluster readiness endpoint"
+  wait_http_ok "${ENDPOINT}/health/ready"
+
+  log_info "Waiting for S3 API readiness via mc alias/list"
+  wait_s3_api_ready
+}
+
+cluster_network() {
+  echo "${PROJECT_NAME}_rustfs-cluster-net"
+}
+
+run_mc_in_network() {
+  docker run --rm --network "$(cluster_network)" "${MC_IMAGE}" "$@"
+}
+
+run_mc_shell_in_network() {
+  docker run --rm --network "$(cluster_network)" --entrypoint /bin/sh "${MC_IMAGE}" -lc "$1"
+}
+
+run_warp_in_network() {
+  docker run --rm --network "$(cluster_network)" -v "${OUT_DIR}:/out" "${WARP_IMAGE}" "$@"
+}
+
+wait_s3_api_ready() {
+  local start now
+  start="$(date +%s)"
+
+  while true; do
+    if run_mc_in_network alias set rustfs "http://node1:9000" "${RUSTFS_ACCESS_KEY}" "${RUSTFS_SECRET_KEY}" >/dev/null 2>&1; then
+      if run_mc_in_network ls rustfs >/dev/null 2>&1; then
+        return 0
+      fi
+    fi
+
+    now="$(date +%s)"
+    if (( now - start >= S3_READY_TIMEOUT_SECS )); then
+      log_error "timed out waiting for S3 API readiness via mc alias/list"
+      return 1
+    fi
+
+    sleep 2
+  done
+}
+
+collect_cluster_info() {
+  local node
+
+  for node in node1 node2 node3 node4; do
+    log_info "Collecting rustfs info from ${node}"
+    compose exec -T "${node}" rustfs info --all --json > "${OUT_DIR}/${node}-info.json"
+  done
+}
+
+run_plain_smoke() {
+  local prefix
+  prefix="plain-smoke-$(date +%s)"
+
+  log_info "Preparing bucket ${BUCKET}"
+  run_mc_in_network alias set rustfs "http://node1:9000" "${RUSTFS_ACCESS_KEY}" "${RUSTFS_SECRET_KEY}" >/dev/null
+  run_mc_in_network ls "rustfs/${BUCKET}" >/dev/null 2>&1 || run_mc_in_network mb "rustfs/${BUCKET}" >/dev/null
+
+  log_info "Running plain smoke write/read/delete: objects=${SMOKE_OBJECTS} size_mb=${SMOKE_OBJECT_SIZE_MB} parallelism=${SMOKE_PARALLELISM}"
+  run_mc_shell_in_network "
+    set -euo pipefail
+    i=1
+    while [ \"\$i\" -le \"${SMOKE_OBJECTS}\" ]; do
+      end=\$((i + ${SMOKE_PARALLELISM} - 1))
+      [ \"\$end\" -le \"${SMOKE_OBJECTS}\" ] || end='${SMOKE_OBJECTS}'
+      j=\"\$i\"
+      while [ \"\$j\" -le \"\$end\" ]; do
+        (
+          dd if=/dev/urandom bs='${SMOKE_OBJECT_SIZE_MB}M' count=1 2>/dev/null | \
+            mc pipe 'rustfs/${BUCKET}/${prefix}/obj-\${j}' >/dev/null
+        ) &
+        j=\$((j + 1))
+      done
+      wait
+      i=\$((end + 1))
+    done
+    mc rm --recursive --force 'rustfs/${BUCKET}/${prefix}' >/dev/null
+  "
+}
+
+run_warp_multipart() {
+  log_info "Running warp multipart-put against ${WARP_HOST}"
+  run_warp_in_network multipart-put \
+    --host "${WARP_HOST}" \
+    --access-key "${RUSTFS_ACCESS_KEY}" \
+    --secret-key "${RUSTFS_SECRET_KEY}" \
+    --bucket "${BUCKET}-warp" \
+    --lookup path \
+    --duration "${WARP_DURATION}" \
+    --concurrent "${WARP_CONCURRENT}" \
+    --parts "${WARP_PARTS}" \
+    --part.size "${WARP_PART_SIZE}" \
+    --part.concurrent "${WARP_PART_CONCURRENT}" \
+    --benchdata /out/warp.csv.zst \
+    --analyze.v \
+    --no-color \
+    | tee "${OUT_DIR}/warp.log"
+}
+
+count_matches() {
+  local pattern="$1"
+  shift
+  local total=0
+  local file count
+
+  for file in "$@"; do
+    count="$(grep -cE "${pattern}" "${file}" 2>/dev/null || true)"
+    total=$((total + ${count:-0}))
+  done
+
+  echo "${total}"
+}
+
+collect_cluster_logs() {
+  local node
+
+  for node in node1 node2 node3 node4; do
+    compose logs --no-color "${node}" > "${OUT_DIR}/${node}.log" || true
+  done
+}
+
+summarize_results() {
+  local warp_log summary
+  local warp_errors quorum_not_reached connection_refused storage_insufficient
+  local lock_timeout erasure_write_quorum readiness_probe_failed liveness_probe_failed
+  local diag_read_parts diag_complete_part diag_rename_part
+  local startup_range_oob startup_volume_not_found startup_remote_network startup_remote_faulty startup_remote_lock_rpc
+
+  warp_log="${OUT_DIR}/warp.log"
+  summary="${OUT_DIR}/summary.txt"
+
+  warp_errors="$(count_matches 'warp: <ERROR>' "${warp_log}")"
+  quorum_not_reached="$(count_matches 'Quorum not reached' "${warp_log}")"
+  connection_refused="$(count_matches 'connection refused' "${warp_log}")"
+  storage_insufficient="$(count_matches 'Storage resources are insufficient for the write operation' "${warp_log}")"
+  lock_timeout="$(count_matches 'Lock acquisition timeout' "${warp_log}")"
+  erasure_write_quorum="$(count_matches 'erasure write quorum' "${warp_log}")"
+  readiness_probe_failed="$(count_matches 'readiness_probe_failed' "${warp_log}")"
+  liveness_probe_failed="$(count_matches 'liveness_probe_failed' "${warp_log}")"
+
+  diag_read_parts="$(count_matches 'issue3031_read_parts_part_quorum' "${OUT_DIR}"/node*.log)"
+  diag_complete_part="$(count_matches 'issue3031_complete_part_error' "${OUT_DIR}"/node*.log)"
+  diag_rename_part="$(count_matches 'issue3031_rename_part_context' "${OUT_DIR}"/node*.log)"
+
+  startup_range_oob="$(count_matches 'Range \\[0, 4\\) is out of bounds' "${OUT_DIR}"/node*.log)"
+  startup_volume_not_found="$(count_matches 'volume not found' "${OUT_DIR}"/node*.log)"
+  startup_remote_network="$(count_matches 'Remote disk operation returned a network-like error' "${OUT_DIR}"/node*.log)"
+  startup_remote_faulty="$(count_matches 'Remote disk marked faulty after timeout' "${OUT_DIR}"/node*.log)"
+  startup_remote_lock_rpc="$(count_matches 'Evicting cached remote lock connection after RPC failure' "${OUT_DIR}"/node*.log)"
+
+  cat > "${summary}" <<EOF
+issue=3031
+endpoint=${ENDPOINT}
+bucket=${BUCKET}
+warp_duration=${WARP_DURATION}
+warp_concurrent=${WARP_CONCURRENT}
+warp_parts=${WARP_PARTS}
+warp_part_size=${WARP_PART_SIZE}
+warp_part_concurrent=${WARP_PART_CONCURRENT}
+
+warp_errors=${warp_errors}
+quorum_not_reached=${quorum_not_reached}
+connection_refused=${connection_refused}
+storage_insufficient=${storage_insufficient}
+lock_timeout=${lock_timeout}
+erasure_write_quorum=${erasure_write_quorum}
+readiness_probe_failed=${readiness_probe_failed}
+liveness_probe_failed=${liveness_probe_failed}
+
+issue3031_read_parts_part_quorum=${diag_read_parts}
+issue3031_complete_part_error=${diag_complete_part}
+issue3031_rename_part_context=${diag_rename_part}
+
+startup_range_oob=${startup_range_oob}
+startup_volume_not_found=${startup_volume_not_found}
+startup_remote_network_error=${startup_remote_network}
+startup_remote_faulty=${startup_remote_faulty}
+startup_remote_lock_rpc=${startup_remote_lock_rpc}
+EOF
+
+  log_info "Summary written to ${summary}"
+  cat "${summary}"
+}
+
+main() {
+  parse_args "$@"
+  require_cmd docker
+  require_cmd curl
+  require_cmd grep
+  require_cmd tee
+
+  mkdir -p "${OUT_DIR}"
+  trap cleanup EXIT INT TERM
+
+  export RUSTFS_IMAGE
+  export RUSTFS_ACCESS_KEY
+  export RUSTFS_SECRET_KEY
+  export RUSTFS_UNSAFE_BYPASS_DISK_CHECK
+  export RUSTFS_ISSUE3031_DIAG_ENABLE
+  export RUSTFS_OBS_LOG_STDOUT_ENABLED
+  export RUSTFS_OBS_USE_STDOUT
+  export RUSTFS_OBS_LOGGER_LEVEL
+  export RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT
+  export RUSTFS_LOCK_ACQUIRE_TIMEOUT
+
+  build_image_if_needed
+  start_cluster
+  collect_cluster_info
+  run_plain_smoke
+  run_warp_multipart
+  collect_cluster_logs
+  summarize_results
+}
+
+main "$@"
Author	SHA1	Message	Date
wood	58ac19f7a4	fix(sse): optimize is_encrypted for old metadata compatibility (#3113 ) Signed-off-by: w0od <dingboning02@163.com> Co-authored-by: houseme <housemecn@gmail.com>	2026-05-29 11:59:08 +00:00
安正超	ac97ceb744	fix(config): restore default credential startup (#3114 ) * fix(config): restore default credential startup * fix: align e2e credentials with server env * fix(config): restore default credential consistency --------- Co-authored-by: houseme <housemecn@gmail.com>	2026-05-29 11:52:46 +00:00
houseme	5d74637968	fix(utils): tolerate bavail greater than bfree on Linux (#3119 ) * fix(utils): tolerate bavail greater than bfree on Linux Treat f_bavail > f_bfree as a compatibility edge case instead of fatal corruption during startup disk info probing. - keep corruption checks for other invalid counter relationships - log a warning and clamp reserved blocks to 0 when bavail exceeds bfree - add linux unit coverage for the compatibility and overflow paths Refs #3025 * fix(utils): cap bavail to bfree for Linux statfs edge * fix(utils): rate-limit Linux statfs compatibility warning * test(utils): cover Linux statfs capacity math Expand Linux statfs capacity math tests around normal, equal, zero, and fully-free block counter combinations. Refs #3025 * fix(utils): avoid statfs warning guard reallocations * test(utils): align statfs reserved block expectations * fmt * fix(utils): avoid warning guard path allocation Check the warn-once set before allocating a PathBuf so repeated bavail/bfree compatibility warnings stay low-overhead. Refs #3025	2026-05-29 11:20:47 +00:00
houseme	9f78cd3896	chore(deps): bump workspace dependencies (#3118 ) * chore(deps): bump workspace dependencies - bump direct dependency versions in Cargo.toml\n- refresh Cargo.lock to resolve updated crates * fix(deps): avoid pyroscope and pprof linker collision - downgrade pyroscope from 2.0.6 to 2.0.5\n- keep a single pprof implementation path via pprof-pyroscope-fork\n- fix duplicate perf_signal_handler during test linking	2026-05-29 09:36:19 +00:00
houseme	2b82432f9e	fix(ecstore): send valid ping body in remote locker (#3112 ) * fix(ecstore): send valid ping body in remote locker Build ping requests with a flatbuffer payload so health checks remain compatible with the ping response parser after restart. * fix(bench): use multi-host warp target during failover Normalize comma-separated warp host lists in run_object_batch_bench and let four-node failover bench pass BENCH_WARP_HOSTS so rolling restart does not pin load to a single restarting node. * feat(health): add compat health probes with busy/KMS checks - Add /health/live liveness probe endpoint - Add busy protection (429) for readiness probes, gated by RUSTFS_HEALTH_COMPAT_BUSY_CHECK_ENABLE - Add KMS readiness check for /health/ready, gated by RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE - Add lock quorum status caching with TTL to reduce RPC pressure - Consolidate health response building into build_health_response_parts - Register /health/live in console router and readiness gate - Remove MinIO references from newly added health code * fix(health): decouple kms readiness from lock quorum	2026-05-29 08:02:50 +00:00
GatewayJ	c257043b63	fix(iam): serialize IAM cache writes (#3105 ) * fix(iam): serialize IAM cache writes * fix(iam): timestamp rebuilt group memberships * fix(iam): publish cache updates atomically * fix(iam): reuse policy cache snapshots * fix(iam): commit missing user notification cache updates atomically * fix(iam): remove unused cache membership rebuild wrapper --------- Co-authored-by: 季宏伟 <jihongwei@jihongweis-MacBook-Pro.local> Co-authored-by: houseme <housemecn@gmail.com>	2026-05-29 08:02:42 +00:00
houseme	ede813070f	fix(site-replication): refresh TLS peer client and tls inspect alias (#3109 ) * fix(site-replication): refresh peer TLS client cache - rebuild site-replication peer reqwest client when outbound TLS generation changes\n- keep cached client/failed state per generation for low-overhead requests\n- accept --tls-path as alias of tls inspect --path for operator compatibility\n- add targeted parser/cache tests\n\nRefs #2723 * docs(issue-2723): add private-ca cert rotation retest - add executable retest checklist for private CA site-replication flows\n- cover tls inspect compatibility (--path and --tls-path)\n- include post-rotation and post-restart pass criteria\n\nRefs #2723 * test(site-replication): isolate global TLS cache test - serialize generation-sensitive peer TLS cache test\n- snapshot and restore global outbound TLS generation and static cache\n- avoid order-dependent interactions with parallel test execution\n\nRefs #2723 * chore: remove docs file from tracking, keep locally The docs directory is already in .gitignore but this file was previously committed. Remove it from git index to prevent docs content from being included in the PR. Co-Authored-By: heihutu <heihutu@gmail.com> * test(site-replication): cover ready client cache hit path Add a focused unit test for unchanged-generation Ready cache entries in site_replication_peer_client_cache_hit to validate steady-state client reuse behavior highlighted in PR #3109 review feedback. --------- Co-authored-by: loverustfs <hello@rustfs.com> Co-authored-by: heihutu <heihutu@gmail.com>	2026-05-29 07:17:55 +00:00
安正超	b436272642	fix: tolerate stalled listing readers after quorum (#3110 ) Co-authored-by: loverustfs <hello@rustfs.com>	2026-05-29 08:57:57 +08:00
houseme	1de0ba916d	fix(ecstore): reduce restart-time startup race noise (#3108 ) Treat empty ping bodies as liveness probes, add a startup cleanup barrier for early walk_dir calls, and delay immediate background cleanup/capacity timers to reduce transient restart-time VolumeNotFound noise. Also downgrade expected missing-path producer results during startup from generic errors to warnings while preserving existing storage semantics.	2026-05-28 15:14:30 +00:00
houseme	088c4bda43	fix(ecstore): harden issue3031 multipart validation path (#3106 ) * fix(ecstore): harden issue3031 multipart validation path - clear stale multipart part destinations before rename fan-out - add repeated part overwrite regression coverage - reduce remote disk startup false-fault escalation to suspect-first - refine remote locker diagnostics and lower scanner leader-lock log noise - add a dedicated 4-node issue3031 docker validation script * refactor(admin): inline console version json macro - drop the unused serde_json::json import in admin console - call serde_json::json! inline in version_handler - keep the console version response behavior unchanged * fix(remote-disk): recover suspect health on probe success - record probe success during remote disk health checks so suspect drives recover - use async_with_vars for the remote disk health probe test - make the missing-listener test assert the state transition more robustly	2026-05-28 14:26:31 +00:00
GatewayJ	8d20e89bf8	fix(iam): avoid stale cache replacement on walk errors (#3094 ) * fix(iam): avoid stale cache replacement on walk errors * fix(iam): guard full reload cache commits --------- Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: loverustfs <hello@rustfs.com> Co-authored-by: 季宏伟 <jihongwei@jihongweis-MacBook-Pro.local>	2026-05-28 09:54:06 +00:00
houseme	28bac7fbd6	chore(release): prepare 1.0.0-beta.6 (#3104 ) * chore(release): prepare 1.0.0-beta.6 * ci(nix): harden flaky crate fetch handling * ci(nix): drop magic cache and force fallback * ci(nix): set explicit user-agent for crate fetch * ci(nix): adopt determinate nix workflow stack * ci(nix): add nix user-agent suffix for fetches * ci(nix): add flakehub cache and align determinate actions * ci(nix): pin determinate actions to release tags * ci(nix): disable flakehub auth path in CI cache * ci(nix): restore stable magic cache baseline * ci(nix): trust local magic cache substituter * ci(nix): stop forcing Node24 for JS actions * ci(nix): drop manual localhost cache config * ci(nix): adopt latest determinate flakehub stack * ci(nix): record latest determinate workflow state	2026-05-28 09:21:16 +00:00
cxymds	527faad575	docs(readme): add Discord community links (#3103 )	2026-05-28 12:22:17 +08:00
houseme	dd1ffbaee7	fix(lock): isolate retry attempt lock ids (#3102 ) * fix(lock): isolate retry attempt lock ids Generate a fresh lock id for each distributed lock retry attempt so late pending cleanup from an earlier attempt cannot release a lock acquired by a later retry. Also add a regression test covering late-success cleanup versus retried acquisition. * style(lock): normalize retry race regression test Normalize the distributed lock retry race regression test formatting without changing behavior. * test(lock): remove quorum-order assumption Make the retry race regression test assert the safety property without assuming which delayed client enters the retry quorum first.	2026-05-28 03:56:12 +00:00
GatewayJ	247973f34c	fix(lifecycle): make transition worker resize nonblocking (#3090 ) Co-authored-by: 季宏伟 <jihongwei@jihongweis-MacBook-Pro.local> Co-authored-by: houseme <housemecn@gmail.com>	2026-05-28 01:36:08 +00:00
安正超	d9c683decc	fix: retry namespace lock quorum contention (#3098 ) * fix: retry namespace lock quorum contention * fix(lock): treat remote lock rpc failures as hard * fix(lock): classify lock contention timeout and avoid hard rollback blocks * fix(lock): handle namespace lock quorum timeouts * fix(lock): refine quorum contention handling * fix(lock): refine unrecoverable quorum handling logs * fix: address namespace lock quorum review feedback * test: stabilize batch quorum timing tests * fix(lock): classify remote quorum failures * fix: drop remote lock timeout grace period * fix: satisfy clippy on lock quorum error * fix: classify remote lock rpc timeouts as hard failures * fix: fast fail impossible lock quorum attempts * fix: retry on transient timeout with pending cleanup * fix: evict remote lock timeout connections * refactor: avoid duplicate timeout eviction warning --------- Co-authored-by: houseme <housemecn@gmail.com>	2026-05-28 01:35:00 +00:00
overtrue	f77e979e4a	fix: satisfy clippy for Snowball traversal test	2026-05-28 09:16:00 +08:00
overtrue	95836a0a4d	fix: reject Snowball extract path traversal	2026-05-28 08:32:13 +08:00
Henry Guo	0c52334480	fix(lock): retry transient distributed lock timeouts (#3101 ) * fix(lock): retry transient distributed lock timeouts * fix(lock): avoid retry during pending lock cleanup * fix(lock): preserve acquire timeout budget --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>	2026-05-27 17:55:46 +00:00
Henry Guo	f8e6fc1f10	fix(ecstore): offload erasure encoding from async workers (#3099 ) * fix(ecstore): offload erasure encoding from async workers * fix(ecstore): reuse encode buffers in blocking tasks --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>	2026-05-27 17:54:05 +00:00
houseme	5dfc1b5f07	fix(readiness): gate on lock quorum health (#3100 ) * fix(readiness): gate on lock quorum health Include distributed lock quorum in runtime readiness decisions and expose the signal in health responses. Map namespace lock quorum failures to ServiceUnavailable instead of InternalError so clients can safely retry while keeping quorum enforcement unchanged. * fix(ecstore): restore namespace lock error decoding Add the missing from_u32 arm for NamespaceLockQuorumUnavailable and cover the numeric roundtrip in error tests.	2026-05-27 17:12:52 +00:00
houseme	4648de9e62	chore(deploy): refine systemd and nixos service docs (#3096 ) * fix(deploy): simplify systemd service startup * docs(deploy): add nixos systemd service example * docs(deploy): add nixos service guide * chore(deps): update pulsar and pyroscope * docs(deploy): refine nixos service guidance * fix(deploy): use explicit rustfs server entrypoint	2026-05-27 14:47:47 +00:00
Derek Ditch	11e97951fd	fix(helm): add LoadBalancer service type support (#3049 ) The service template only handled ClusterIP and NodePort via if/else-if branches. When service.type=LoadBalancer was set, no branch matched and the type field was omitted from the rendered Service, causing Kubernetes to silently default to ClusterIP. Add the missing LoadBalancer branch with support for: - loadBalancerIP: request a specific IP (e.g. Cilium LB-IPAM, MetalLB) - loadBalancerClass: select a specific LB implementation - loadBalancerSourceRanges: restrict client source IPs This is particularly useful for bare-metal / on-prem clusters using software load balancers (Cilium LB-IPAM, MetalLB, kube-vip) where a stable routable IP is preferable to an Ingress for S3 clients. Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: 安正超 <anzhengchao@gmail.com> Co-authored-by: cxymds <Cxymds@qq.com> Co-authored-by: majinghe <42570491+majinghe@users.noreply.github.com>	2026-05-27 14:35:09 +00:00
安正超	f53eb4ad44	fix: reject invalid multipart part numbers (#3091 ) Co-authored-by: houseme <housemecn@gmail.com>	2026-05-27 08:09:59 +00:00
houseme	ceffe21f75	feat(tls): add inspect command for TLS layouts (#3092 )	2026-05-27 07:23:58 +00:00
houseme	909f0d57fb	feat: improve degraded readiness reporting and shutdown handling (#3089 ) * fix(server): unify runtime readiness and shutdown flow * refactor(server): decouple readiness shared types * refactor(server): keep readiness types crate-private * refactor(server): await protocol shutdown handles * fix(server): bypass cached startup readiness * feat(health): expose degraded readiness reasons * fix(health): preserve raw IAM readiness in degraded reports	2026-05-27 03:25:28 +00:00