fix: restore default CORS fallback and STS object ACL ownership (#2053 )

Co-authored-by: houseme <housemecn@gmail.com>
fix sse-options (#2056 )
2026-07-30 01:58:59 +00:00 · 2026-03-03 01:08:50 +08:00 · 2026-03-03 01:08:37 +08:00 · 2026-03-03 00:44:23 +08:00 · 2026-03-02 23:59:34 +08:00 · 2026-03-02 19:48:12 +08:00
522 changed files with 69473 additions and 33595 deletions
@@ -1,5 +1,7 @@
 ## —— Tests and e2e test ---------------------------------------------------------------------------

+TEST_THREADS ?= 1
+
 .PHONY: test
 test: core-deps test-deps ## Run all tests
 	@echo "🧪 Running tests..."
@@ -7,7 +9,7 @@ test: core-deps test-deps ## Run all tests
 		cargo nextest run --all --exclude e2e_test; \
 	else \
 		echo "ℹ️ cargo-nextest not found; falling back to 'cargo test'"; \
-		cargo test --workspace --exclude e2e_test -- --nocapture; \
+		cargo test --workspace --exclude e2e_test -- --nocapture --test-threads="$(TEST_THREADS)"; \
 	fi
 	cargo test --all --doc

@@ -17,4 +19,4 @@ e2e-server: ## Run e2e-server tests

 .PHONY: probe-e2e
 probe-e2e: ## Probe e2e tests
-	sh $(shell pwd)/scripts/probe.sh
+	sh $(shell pwd)/scripts/probe.sh
@@ -1,261 +1,131 @@
-# RustFS Docker Images
+# RustFS Docker Infrastructure

-This directory contains Docker configuration files and supporting infrastructure for building and running RustFS container images.
+This directory contains the complete Docker infrastructure for building, deploying, and monitoring RustFS. It provides ready-to-use configurations for development, testing, and production-grade observability.

-## 📁 Directory Structure
+## 📂 Directory Structure

-```
-rustfs/
-├── Dockerfile           # Production image (Alpine + pre-built binaries)
-├── Dockerfile.source    # Development image (Debian + source build)
-├── docker-buildx.sh     # Multi-architecture build script
-├── Makefile             # Build automation with simplified commands
-└── .docker/             # Supporting infrastructure
-    ├── observability/   # Monitoring and observability configs
-    ├── compose/         # Docker Compose configurations
-    ├── mqtt/            # MQTT broker configs
-    └── openobserve-otel/ # OpenObserve + OpenTelemetry configs
-```
+| Directory | Description | Status |
+| :--- | :--- | :--- |
+| **[`observability/`](observability/README.md)** | **[RECOMMENDED]** Full-stack observability (Prometheus, Grafana, Tempo, Loki). | ✅ Production-Ready |
+| **[`compose/`](compose/README.md)** | Specialized setups (e.g., 4-node distributed cluster testing). | ⚠️ Testing Only |
+| **[`mqtt/`](mqtt/README.md)** | EMQX Broker configuration for MQTT integration testing. | 🧪 Development |
+| **[`openobserve-otel/`](openobserve-otel/README.md)** | Alternative lightweight observability stack using OpenObserve. | 🔄 Alternative |

-## 🎯 Image Variants
+---

-### Core Images
+## 📄 Root Directory Files

-| Image | Base OS | Build Method | Size | Use Case |
-|-------|---------|--------------|------|----------|
-| `production` (default) | Alpine 3.18 | GitHub Releases | Smallest | Production deployment |
-| `source` | Debian Bookworm | Source build | Medium | Custom builds with cross-compilation |
-| `dev` | Debian Bookworm | Development tools | Large | Interactive development |
+The following files in the project root are essential for Docker operations:

-## 🚀 Usage Examples
+### Build Scripts & Dockerfiles

-### Quick Start (Production)
+| File | Description | Usage |
+| :--- | :--- | :--- |
+| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
+| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
+| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
+| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |

+### Docker Compose Configurations
+
+| File | Description | Usage |
+| :--- | :--- | :--- |
+| **`docker-compose.yml`** | **Main Development Setup**<br>Comprehensive setup with profiles for development, observability, and proxying. | `docker compose up -d`<br>`docker compose --profile observability up -d` |
+| **`docker-compose-simple.yml`** | **Quick Start Setup**<br>Minimal configuration running a single RustFS instance with 4 volumes. Perfect for first-time users. | `docker compose -f docker-compose-simple.yml up -d` |
+
+---
+
+## 🌟 Observability Stack (Recommended)
+
+Located in: [`.docker/observability/`](observability/README.md)
+
+We provide a comprehensive, industry-standard observability stack designed for deep insights into RustFS performance. This is the recommended setup for both development and production monitoring.
+
+### Components
+- **Metrics**: Prometheus (Collection) + Grafana (Visualization)
+- **Traces**: Tempo (Storage) + Jaeger (UI)
+- **Logs**: Loki
+- **Ingestion**: OpenTelemetry Collector
+
+### Key Features
+- **Full Persistence**: All metrics, logs, and traces are saved to Docker volumes, ensuring no data loss on restarts.
+- **Correlation**: Seamlessly jump between Logs, Traces, and Metrics in Grafana.
+- **High Performance**: Optimized configurations for batching, compression, and memory management.
+
+### Quick Start
 ```bash
-# Default production image (Alpine + GitHub Releases)
-docker run -p 9000:9000 rustfs/rustfs:latest
-
-# Specific version
-docker run -p 9000:9000 rustfs/rustfs:1.2.3
+cd .docker/observability
+docker compose up -d
 ```

-### Complete Tag Strategy Examples
+---

+## 🧪 Specialized Environments
+
+Located in: [`.docker/compose/`](compose/README.md)
+
+These configurations are tailored for specific testing scenarios that require complex topologies.
+
+### Distributed Cluster (4-Nodes)
+Simulates a real-world distributed environment with 4 RustFS nodes running locally.
 ```bash
-# Stable Releases
-docker run rustfs/rustfs:1.2.3           # Main version (production)
-docker run rustfs/rustfs:1.2.3-production # Explicit production variant
-docker run rustfs/rustfs:1.2.3-source   # Source build variant
-docker run rustfs/rustfs:latest          # Latest stable
-
-# Prerelease Versions
-docker run rustfs/rustfs:1.3.0-alpha.2  # Specific alpha version
-docker run rustfs/rustfs:alpha           # Latest alpha
-docker run rustfs/rustfs:beta            # Latest beta
-docker run rustfs/rustfs:rc              # Latest release candidate
-
-# Development Versions
-docker run rustfs/rustfs:dev             # Latest main branch development
-docker run rustfs/rustfs:dev-13e4a0b     # Specific commit
-docker run rustfs/rustfs:dev-latest      # Latest development
-docker run rustfs/rustfs:main-latest     # Main branch latest
+docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
 ```

-### Development Environment
-
+### Integrated Observability Test
+A self-contained environment running 4 RustFS nodes alongside the full observability stack. Useful for end-to-end testing of telemetry.
 ```bash
-# Quick setup using Makefile (recommended)
-make docker-dev-local     # Build development image locally
-make dev-env-start        # Start development container
-
-# Manual Docker commands
-docker run -it -v $(pwd):/workspace -p 9000:9000 rustfs/rustfs:latest-dev
-
-# Build from source locally
-docker build -f Dockerfile.source -t rustfs:custom .
-
-# Development with hot reload
-docker-compose up rustfs-dev
+docker compose -f .docker/compose/docker-compose.observability.yaml up -d
 ```

-## 🏗️ Build Arguments and Scripts
+---

-### Using Makefile Commands (Recommended)
+## 📡 MQTT Integration

-The easiest way to build images using simplified commands:
+Located in: [`.docker/mqtt/`](mqtt/README.md)

+Provides an EMQX broker for testing RustFS MQTT features.
+
+### Quick Start
 ```bash
-# Development images (build from source)
-make docker-dev-local             # Build for local use (single arch)
-make docker-dev                   # Build multi-arch (for CI/CD)
-make docker-dev-push REGISTRY=xxx # Build and push to registry
-
-# Production images (using pre-built binaries)
-make docker-buildx                # Build multi-arch production images
-make docker-buildx-push           # Build and push production images
-make docker-buildx-version VERSION=v1.0.0  # Build specific version
-
-# Development environment
-make dev-env-start                # Start development container
-make dev-env-stop                 # Stop development container
-make dev-env-restart              # Restart development container
-
-# Help
-make help-docker                  # Show all Docker-related commands
+cd .docker/mqtt
+docker compose up -d
 ```
+- **Dashboard**: [http://localhost:18083](http://localhost:18083) (Default: `admin` / `public`)
+- **MQTT Port**: `1883`

-### Using docker-buildx.sh (Advanced)
+---

-For direct script usage and advanced scenarios:
+## 👁️ Alternative: OpenObserve

+Located in: [`.docker/openobserve-otel/`](openobserve-otel/README.md)
+
+For users preferring a lightweight, all-in-one solution, we support OpenObserve. It combines logs, metrics, and traces into a single binary and UI.
+
+### Quick Start
 ```bash
-# Build latest version for all architectures
-./docker-buildx.sh
-
-# Build and push to registry
-./docker-buildx.sh --push
-
-# Build specific version
-./docker-buildx.sh --release v1.2.3
-
-# Build and push specific version
-./docker-buildx.sh --release v1.2.3 --push
+cd .docker/openobserve-otel
+docker compose up -d
 ```

-### Manual Docker Builds
+---

-All images support dynamic version selection:
+## 🔧 Common Operations

+### Cleaning Up
+To stop all containers and remove volumes (**WARNING**: deletes all persisted data):
 ```bash
-# Build production image with latest release
-docker build --build-arg RELEASE="latest" -t rustfs:latest .
-
-# Build from source with specific target
-docker build -f Dockerfile.source \
-  --build-arg TARGETPLATFORM="linux/amd64" \
-  -t rustfs:source .
-
-# Development build
-docker build -f Dockerfile.source -t rustfs:dev .
+docker compose down -v
 ```

-## 🔧 Binary Download Sources
-
-### Unified GitHub Releases
-
-The production image downloads from GitHub Releases for reliability and transparency:
-
- ✅ **production** → GitHub Releases API with automatic latest detection
- ✅ **Checksum verification** → SHA256SUMS validation when available
- ✅ **Multi-architecture** → Supports amd64 and arm64
-
-### Source Build
-
-The source variant compiles from source code with advanced features:
-
- 🔧 **Cross-compilation** → Supports multiple target platforms via `TARGETPLATFORM`
- ⚡ **Build caching** → sccache for faster compilation
- 🎯 **Optimized builds** → Release optimizations with LTO and symbol stripping
-
-## 📋 Architecture Support
-
-All variants support multi-architecture builds:
-
- **linux/amd64** (x86_64)
- **linux/arm64** (aarch64)
-
-Architecture is automatically detected during build using Docker's `TARGETARCH` build argument.
-
-## 🔐 Security Features
-
- **Checksum Verification**: Production image verifies SHA256SUMS when available
- **Non-root User**: All images run as user `rustfs` (UID 1000)
- **Minimal Runtime**: Production image only includes necessary dependencies
- **Secure Defaults**: No hardcoded credentials or keys
-
-## 🛠️ Development Workflow
-
-### Quick Start with Makefile (Recommended)
-
+### Viewing Logs
+To follow logs for a specific service:
 ```bash
-# 1. Start development environment
-make dev-env-start
-
-# 2. Your development container is now running with:
-#    - Port 9000 exposed for RustFS
-#    - Port 9010 exposed for admin console
-#    - Current directory mounted as /workspace
-
-# 3. Stop when done
-make dev-env-stop
+docker compose logs -f [service_name]
 ```

-### Manual Development Setup
-
+### Checking Status
+To see the status of all running containers:
 ```bash
-# Build development image from source
-make docker-dev-local
-
-# Or use traditional Docker commands
-docker build -f Dockerfile.source -t rustfs:dev .
-
-# Run with development tools
-docker run -it -v $(pwd):/workspace -p 9000:9000 rustfs:dev bash
-
-# Or use docker-compose for complex setups
-docker-compose up rustfs-dev
+docker compose ps
 ```
-
-### Common Development Tasks
-
-```bash
-# Build and test locally
-make build                        # Build binary natively
-make docker-dev-local            # Build development Docker image
-make test                        # Run tests
-make fmt                         # Format code
-make clippy                      # Run linter
-
-# Get help
-make help                        # General help
-make help-docker                 # Docker-specific help
-make help-build                  # Build-specific help
-```
-
-## 🚀 CI/CD Integration
-
-The project uses GitHub Actions for automated multi-architecture Docker builds:
-
-### Automated Builds
-
- **Tags**: Automatic builds triggered on version tags (e.g., `v1.2.3`)
- **Main Branch**: Development builds with `dev-latest` and `main-latest` tags
- **Pull Requests**: Test builds without registry push
-
-### Build Variants
-
-Each build creates three image variants:
-
- `rustfs/rustfs:v1.2.3` (production - Alpine-based)
- `rustfs/rustfs:v1.2.3-source` (source build - Debian-based)
- `rustfs/rustfs:v1.2.3-dev` (development - Debian-based with tools)
-
-### Manual Builds
-
-Trigger custom builds via GitHub Actions:
-
-```bash
-# Use workflow_dispatch to build specific versions
-# Available options: latest, main-latest, dev-latest, v1.2.3, dev-abc123
-```
-
-## 📦 Supporting Infrastructure
-
-The `.docker/` directory contains supporting configuration files:
-
- **observability/** - Prometheus, Grafana, OpenTelemetry configs
- **compose/** - Multi-service Docker Compose setups
- **mqtt/** - MQTT broker configurations
- **openobserve-otel/** - Log aggregation and tracing setup
-
-See individual README files in each subdirectory for specific usage instructions.
@@ -1,80 +1,44 @@
-# Docker Compose Configurations
+# Specialized Docker Compose Configurations

-This directory contains specialized Docker Compose configurations for different use cases.
+This directory contains specialized Docker Compose configurations for specific testing scenarios.
+
+## ⚠️ Important Note
+
+**For Observability:**
+We **strongly recommend** using the new, fully integrated observability stack located in `../observability/`. It provides a production-ready setup with Prometheus, Grafana, Tempo, Loki, and OpenTelemetry Collector, all with persistent storage and optimized configurations.
+
+The `docker-compose.observability.yaml` in this directory is kept for legacy reference or specific minimal testing needs but is **not** the primary recommended setup.

 ## 📁 Configuration Files

-This directory contains specialized Docker Compose configurations and their associated Dockerfiles, keeping related files organized together.
+### Cluster Testing

-### Main Configuration (Root Directory)
+- **`docker-compose.cluster.yaml`**
+  - **Purpose**: Simulates a 4-node RustFS distributed cluster.
+  - **Use Case**: Testing distributed storage logic, consensus, and failover.
+  - **Nodes**: 4 RustFS instances.
+  - **Storage**: Uses local HTTP endpoints.

- **`../../docker-compose.yml`** - **Default Production Setup**
-  - Complete production-ready configuration
-  - Includes RustFS server + full observability stack
-  - Supports multiple profiles: `dev`, `observability`, `cache`, `proxy`
-  - Recommended for most users
+### Legacy / Minimal Observability

-### Specialized Configurations
-
- **`docker-compose.cluster.yaml`** - **Distributed Testing**
-  - 4-node cluster setup for testing distributed storage
-  - Uses local compiled binaries
-  - Simulates multi-node environment
-  - Ideal for development and cluster testing
-
- **`docker-compose.observability.yaml`** - **Observability Focus**
-  - Specialized setup for testing observability features
-  - Includes OpenTelemetry, Jaeger, Prometheus, Loki, Grafana
-  - Uses `../../Dockerfile.source` for builds
-  - Perfect for observability development
+- **`docker-compose.observability.yaml`**
+  - **Purpose**: A minimal observability setup.
+  - **Status**: **Deprecated**. Please use `../observability/docker-compose.yml` instead.

 ## 🚀 Usage Examples

-### Production Setup
-
-```bash
-# Start main service
-docker-compose up -d
-
-# Start with development profile
-docker-compose --profile dev up -d
-
-# Start with full observability
-docker-compose --profile observability up -d
-```
-
 ### Cluster Testing

-```bash
-# Build and start 4-node cluster (run from project root)
-cd .docker/compose
-docker-compose -f docker-compose.cluster.yaml up -d
-
-# Or run directly from project root
-docker-compose -f .docker/compose/docker-compose.cluster.yaml up -d
-```
-
-### Observability Testing
+To start a 4-node cluster for distributed testing:

 ```bash
-# Start observability-focused environment (run from project root)
-cd .docker/compose
-docker-compose -f docker-compose.observability.yaml up -d
-
-# Or run directly from project root
-docker-compose -f .docker/compose/docker-compose.observability.yaml up -d
+# From project root
+docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
 ```

-## 🔧 Configuration Overview
+### (Deprecated) Minimal Observability

-| Configuration | Nodes | Storage | Observability | Use Case |
-|---------------|-------|---------|---------------|----------|
-| **Main** | 1 | Volume mounts | Full stack | Production |
-| **Cluster** | 4 | HTTP endpoints | Basic | Testing |
-| **Observability** | 4 | Local data | Advanced | Development |
-
-## 📝 Notes
-
- Always ensure you have built the required binaries before starting cluster tests
- The main configuration is sufficient for most use cases
- Specialized configurations are for specific testing scenarios
+```bash
+# From project root
+docker compose -f .docker/compose/docker-compose.observability.yaml up -d
+```
@@ -13,62 +13,126 @@
 # limitations under the License.

 services:
+  # --- Observability Stack ---
+
+  tempo-init:
+    image: busybox:latest
+    command: [ "sh", "-c", "chown -R 10001:10001 /var/tempo" ]
+    volumes:
+      - tempo-data:/var/tempo
+    user: root
+    networks:
+      - rustfs-network
+    restart: "no"
+
+  tempo:
+    image: grafana/tempo:latest
+    user: "10001"
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ../../.docker/observability/tempo.yaml:/etc/tempo.yaml:ro
+      - tempo-data:/var/tempo
+    ports:
+      - "3200:3200" # tempo
+      - "4317"      # otlp grpc
+      - "4318"      # otlp http
+    restart: unless-stopped
+    networks:
+      - rustfs-network
+
  otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.129.1
+    image: otel/opentelemetry-collector-contrib:latest
    environment:
      - TZ=Asia/Shanghai
    volumes:
-      - ../../.docker/observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
+      - ../../.docker/observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
    ports:
-      - 1888:1888
-      - 8888:8888
-      - 8889:8889
-      - 13133:13133
-      - 4317:4317
-      - 4318:4318
-      - 55679:55679
+      - "1888:1888"   # pprof
+      - "8888:8888"   # Prometheus metrics for Collector
+      - "8889:8889"   # Prometheus metrics for application indicators
+      - "13133:13133" # health check
+      - "4317:4317"   # OTLP gRPC
+      - "4318:4318"   # OTLP HTTP
+      - "55679:55679" # zpages
    networks:
      - rustfs-network
+    depends_on:
+      - tempo
+      - jaeger
+      - prometheus
+      - loki
+
  jaeger:
-    image: jaegertracing/jaeger:2.8.0
+    image: jaegertracing/jaeger:latest
    environment:
      - TZ=Asia/Shanghai
+      - SPAN_STORAGE_TYPE=badger
+      - BADGER_EPHEMERAL=false
+      - BADGER_DIRECTORY_VALUE=/badger/data
+      - BADGER_DIRECTORY_KEY=/badger/key
+      - COLLECTOR_OTLP_ENABLED=true
+    volumes:
+      - jaeger-data:/badger
    ports:
-      - "16686:16686"
-      - "14317:4317"
-      - "14318:4318"
+      - "16686:16686" # Web UI
+      - "14269:14269" # Admin/Metrics
    networks:
      - rustfs-network
+
  prometheus:
-    image: prom/prometheus:v3.4.2
+    image: prom/prometheus:latest
    environment:
      - TZ=Asia/Shanghai
    volumes:
-      - ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
    ports:
      - "9090:9090"
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.enable-otlp-receiver'
+      - '--web.enable-remote-write-receiver'
+      - '--enable-feature=promql-experimental-functions'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
    networks:
      - rustfs-network
+
  loki:
-    image: grafana/loki:3.5.1
+    image: grafana/loki:latest
    environment:
      - TZ=Asia/Shanghai
    volumes:
-      - ../../.docker/observability/loki-config.yaml:/etc/loki/local-config.yaml
+      - ../../.docker/observability/loki-config.yaml:/etc/loki/local-config.yaml:ro
+      - loki-data:/loki
    ports:
      - "3100:3100"
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - rustfs-network
+
  grafana:
-    image: grafana/grafana:12.0.2
+    image: grafana/grafana:latest
    ports:
      - "3000:3000" # Web UI
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_SECURITY_ADMIN_USER=admin
      - TZ=Asia/Shanghai
+      - GF_INSTALL_PLUGINS=grafana-pyroscope-datasource
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/home.json
    networks:
      - rustfs-network
+    volumes:
+      - ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+    depends_on:
+      - prometheus
+      - tempo
+      - loki
+
+  # --- RustFS Cluster ---

  node1:
    build:
@@ -79,13 +143,15 @@ services:
      - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
      - RUSTFS_OBS_LOGGER_LEVEL=debug
    platform: linux/amd64
    ports:
-      - "9001:9000" # Map port 9001 of the host to port 9000 of the container
+      - "9001:9000"
    networks:
      - rustfs-network
+    depends_on:
+      - otel-collector

  node2:
    build:
@@ -96,13 +162,15 @@ services:
      - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
      - RUSTFS_OBS_LOGGER_LEVEL=debug
    platform: linux/amd64
    ports:
-      - "9002:9000" # Map port 9002 of the host to port 9000 of the container
+      - "9002:9000"
    networks:
      - rustfs-network
+    depends_on:
+      - otel-collector

  node3:
    build:
@@ -113,13 +181,15 @@ services:
      - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
      - RUSTFS_OBS_LOGGER_LEVEL=debug
    platform: linux/amd64
    ports:
-      - "9003:9000" # Map port 9003 of the host to port 9000 of the container
+      - "9003:9000"
    networks:
      - rustfs-network
+    depends_on:
+      - otel-collector

  node4:
    build:
@@ -130,13 +200,21 @@ services:
      - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
      - RUSTFS_OBS_LOGGER_LEVEL=debug
    platform: linux/amd64
    ports:
-      - "9004:9000" # Map port 9004 of the host to port 9000 of the container
+      - "9004:9000"
    networks:
      - rustfs-network
+    depends_on:
+      - otel-collector
+
+volumes:
+  prometheus-data:
+  tempo-data:
+  loki-data:
+  jaeger-data:

 networks:
  rustfs-network:
@@ -0,0 +1,30 @@
+# MQTT Broker (EMQX)
+
+This directory contains the configuration for running an EMQX MQTT broker, which can be used for testing RustFS's MQTT integration.
+
+## 🚀 Quick Start
+
+To start the EMQX broker:
+
+```bash
+docker compose up -d
+```
+
+## 📊 Access
+
+- **Dashboard**: [http://localhost:18083](http://localhost:18083)
+- **Default Credentials**: `admin` / `public`
+- **MQTT Port**: `1883`
+- **WebSocket Port**: `8083`
+
+## 🛠️ Configuration
+
+The `docker-compose.yml` file sets up a single-node EMQX instance.
+
+- **Persistence**: Data is not persisted by default (for testing).
+- **Network**: Uses the default bridge network.
+
+## 📝 Notes
+
+- This setup is intended for development and testing purposes.
+- For production deployments, please refer to the official [EMQX Documentation](https://www.emqx.io/docs/en/latest/).
@@ -0,0 +1,82 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+worker_processes auto;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+    error_log /var/log/nginx/error.log warn;
+
+    sendfile on;
+    keepalive_timeout 65;
+
+    # RustFS Server Block
+    server {
+        listen 80;
+        server_name localhost;
+
+        # Redirect HTTP to HTTPS (optional, uncomment if SSL is configured)
+        # return 301 https://$host$request_uri;
+
+        location / {
+            proxy_pass http://rustfs:9000;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            # S3 specific headers
+            proxy_set_header X-Amz-Date $http_x_amz_date;
+            proxy_set_header Authorization $http_authorization;
+
+            # Disable buffering for large uploads
+            proxy_request_buffering off;
+            client_max_body_size 0;
+        }
+
+        location /rustfs/console {
+            proxy_pass http://rustfs:9001;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+    }
+
+    # SSL Configuration (Example)
+    # server {
+    #     listen 443 ssl;
+    #     server_name localhost;
+    #
+    #     ssl_certificate /etc/nginx/ssl/server.crt;
+    #     ssl_certificate_key /etc/nginx/ssl/server.key;
+    #
+    #     location / {
+    #         proxy_pass http://rustfs:9000;
+    #         ...
+    #     }
+    # }
+}
@@ -0,0 +1,5 @@
+jaeger-data/*
+loki-data/*
+prometheus-data/*
+tempo-data/*
+grafana-data/*
@@ -1,109 +1,85 @@
-# Observability
+# RustFS Observability Stack

-This directory contains the observability stack for the application. The stack is composed of the following components:
+This directory contains the comprehensive observability stack for RustFS, designed to provide deep insights into application performance, logs, and traces.

- Prometheus v3.2.1
- Grafana 11.6.0
- Loki 3.4.2
- Jaeger 2.4.0
- Otel Collector 0.120.0 # 0.121.0 remove loki
+## Components

-## Prometheus
+The stack is composed of the following best-in-class open-source components:

-Prometheus is a monitoring and alerting toolkit. It scrapes metrics from instrumented jobs, either directly or via an
-intermediary push gateway for short-lived jobs. It stores all scraped samples locally and runs rules over this data to
-either aggregate and record new time series from existing data or generate alerts. Grafana or other API consumers can be
-used to visualize the collected data.
+- **Prometheus** (v2.53.1): The industry standard for metric collection and alerting.
+- **Grafana** (v11.1.0): The leading platform for observability visualization.
+- **Loki** (v3.1.0): A horizontally-scalable, highly-available, multi-tenant log aggregation system.
+- **Tempo** (v2.5.0): A high-volume, minimal dependency distributed tracing backend.
+- **Jaeger** (v1.59.0): Distributed tracing system (configured as a secondary UI/storage).
+- **OpenTelemetry Collector** (v0.104.0): A vendor-agnostic implementation for receiving, processing, and exporting telemetry data.

-## Grafana
+## Architecture

-Grafana is a multi-platform open-source analytics and interactive visualization web application. It provides charts,
-graphs, and alerts for the web when connected to supported data sources.
+1. **Telemetry Collection**: Applications send OTLP (OpenTelemetry Protocol) data (Metrics, Logs, Traces) to the **OpenTelemetry Collector**.
+2. **Processing & Exporting**: The Collector processes the data (batching, memory limiting) and exports it to the respective backends:
+    - **Traces** -> **Tempo** (Primary) & **Jaeger** (Secondary/Optional)
+    - **Metrics** -> **Prometheus** (via scraping the Collector's exporter)
+    - **Logs** -> **Loki**
+3. **Visualization**: **Grafana** connects to all backends (Prometheus, Tempo, Loki, Jaeger) to provide a unified dashboard experience.

-## Loki
+## Features

-Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. It is
-designed to be very cost-effective and easy to operate. It does not index the contents of the logs, but rather a set of
-labels for each log stream.
+- **Full Persistence**: All data (Metrics, Logs, Traces) is persisted to Docker volumes, ensuring no data loss on restart.
+- **Correlation**: Seamless navigation between Metrics, Logs, and Traces in Grafana.
+  - Jump from a Metric spike to relevant Traces.
+  - Jump from a Trace to relevant Logs.
+- **High Performance**: Optimized configurations for batching, compression, and memory management.
+- **Standardized Protocols**: Built entirely on OpenTelemetry standards.

-## Jaeger
+## Quick Start

-Jaeger is a distributed tracing system released as open source by Uber Technologies. It is used for monitoring and
-troubleshooting microservices-based distributed systems, including:
+### Prerequisites

- Distributed context propagation
- Distributed transaction monitoring
- Root cause analysis
- Service dependency analysis
- Performance / latency optimization
+- Docker
+- Docker Compose

-## Otel Collector
+### Deploy

-The OpenTelemetry Collector offers a vendor-agnostic implementation on how to receive, process, and export telemetry
-data. It removes the need to run, operate, and maintain multiple agents/collectors in order to support open-source
-observability data formats (e.g. Jaeger, Prometheus, etc.) sending to one or more open-source or commercial back-ends.
-
-## How to use
-
-To deploy the observability stack, run the following command:
-
- docker latest version
+Run the following command to start the entire stack:

 ```bash
-docker compose -f docker-compose.yml -f docker-compose.override.yml up -d
+docker compose up -d
 ```

- docker compose v2.0.0 or before
+### Access Dashboards
+
+| Service        | URL                                              | Credentials       | Description                    |
+| :------------- | :----------------------------------------------- | :---------------- | :----------------------------- |
+| **Grafana**    | [http://localhost:3000](http://localhost:3000)   | `admin` / `admin` | Main visualization hub.        |
+| **Prometheus** | [http://localhost:9090](http://localhost:9090)   | -                 | Metric queries and status.     |
+| **Jaeger UI**  | [http://localhost:16686](http://localhost:16686) | -                 | Secondary trace visualization. |
+| **Tempo**      | [http://localhost:3200](http://localhost:3200)   | -                 | Tempo status/metrics.          |
+
+## Configuration
+
+### Data Persistence
+
+Data is stored in the following Docker volumes:
+
+- `prometheus-data`: Prometheus metrics
+- `tempo-data`: Tempo traces (WAL and Blocks)
+- `loki-data`: Loki logs (Chunks and Rules)
+- `jaeger-data`: Jaeger traces (Badger DB)
+
+To clear all data:

 ```bash
-docke-compose -f docker-compose.yml -f docker-compose.override.yml up -d
+docker compose down -v
 ```

-To access the Grafana dashboard, navigate to `http://localhost:3000` in your browser. The default username and password
-are `admin` and `admin`, respectively.
-
-To access the Jaeger dashboard, navigate to `http://localhost:16686` in your browser.
-
-To access the Prometheus dashboard, navigate to `http://localhost:9090` in your browser.
-
-## How to stop
-
-To stop the observability stack, run the following command:
-
-```bash
-docker compose -f docker-compose.yml -f docker-compose.override.yml down
-```
-
-## How to remove data
-
-To remove the data generated by the observability stack, run the following command:
-
-```bash
-docker compose -f docker-compose.yml -f docker-compose.override.yml down -v
-```
-
-## How to configure
-
-To configure the observability stack, modify the `docker-compose.override.yml` file. The file contains the following
-
-```yaml
-services:
-  prometheus:
-    environment:
-      - PROMETHEUS_CONFIG_FILE=/etc/prometheus/prometheus.yml
-    volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
-
-  grafana:
-    environment:
-      - GF_SECURITY_ADMIN_PASSWORD=admin
-    volumes:
-      - ./grafana/provisioning:/etc/grafana/provisioning
-```
-
-The `prometheus` service mounts the `prometheus.yml` file to `/etc/prometheus/prometheus.yml`. The `grafana` service
-mounts the `grafana/provisioning` directory to `/etc/grafana/provisioning`. You can modify these files to configure the
-observability stack.
+### Customization

+- **Prometheus**: Edit `prometheus.yml` to add scrape targets or alerting rules.
+- **Grafana**: Dashboards and datasources are provisioned from the `grafana/` directory.
+- **Collector**: Edit `otel-collector-config.yaml` to modify pipelines, processors, or exporters.

+## Troubleshooting

+- **Service Health**: Check the health of services using `docker compose ps`.
+- **Logs**: View logs for a specific service using `docker compose logs -f <service_name>`.
+- **Otel Collector**: Check `http://localhost:13133` for health status and `http://localhost:1888/debug/pprof/` for profiling.
@@ -1,27 +1,85 @@
-## 部署可观测性系统
+# RustFS 可观测性技术栈

-OpenTelemetry Collector 提供了一个厂商中立的遥测数据处理方案，用于接收、处理和导出遥测数据。它消除了为支持多种开源可观测性数据格式（如
-Jaeger、Prometheus 等）而需要运行和维护多个代理/收集器的必要性。
+本目录包含 RustFS 的全面可观测性技术栈，旨在提供对应用程序性能、日志和追踪的深入洞察。

-### 快速部署
+## 组件

-1. 进入 `.docker/observability` 目录
-2. 执行以下命令启动服务：
+该技术栈由以下一流的开源组件组成：
+
+- **Prometheus** (v2.53.1): 行业标准的指标收集和告警工具。
+- **Grafana** (v11.1.0): 领先的可观测性可视化平台。
+- **Loki** (v3.1.0): 水平可扩展、高可用、多租户的日志聚合系统。
+- **Tempo** (v2.5.0): 高吞吐量、最小依赖的分布式追踪后端。
+- **Jaeger** (v1.59.0): 分布式追踪系统（配置为辅助 UI/存储）。
+- **OpenTelemetry Collector** (v0.104.0): 接收、处理和导出遥测数据的供应商无关实现。
+
+## 架构
+
+1.  **遥测收集**: 应用程序将 OTLP (OpenTelemetry Protocol) 数据（指标、日志、追踪）发送到 **OpenTelemetry Collector**。
+2.  **处理与导出**: Collector 处理数据（批处理、内存限制）并将其导出到相应的后端：
+    -   **追踪** -> **Tempo** (主要) & **Jaeger** (辅助/可选)
+    -   **指标** -> **Prometheus** (通过抓取 Collector 的导出器)
+    -   **日志** -> **Loki**
+3.  **可视化**: **Grafana** 连接到所有后端（Prometheus, Tempo, Loki, Jaeger），提供统一的仪表盘体验。
+
+## 特性
+
+-   **完全持久化**: 所有数据（指标、日志、追踪）都持久化到 Docker 卷，确保重启后无数据丢失。
+-   **关联性**: 在 Grafana 中实现指标、日志和追踪之间的无缝导航。
+    -   从指标峰值跳转到相关追踪。
+    -   从追踪跳转到相关日志。
+-   **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
+-   **标准化协议**: 完全基于 OpenTelemetry 标准构建。
+
+## 快速开始
+
+### 前置条件
+
+-   Docker
+-   Docker Compose
+
+### 部署
+
+运行以下命令启动整个技术栈：

 ```bash
-docker compose -f docker-compose.yml  up -d
+docker compose up -d
 ```

-### 访问监控面板
+### 访问仪表盘

-服务启动后，可通过以下地址访问各个监控面板：
+| 服务 | URL | 凭据 | 描述 |
+| :--- | :--- | :--- | :--- |
+| **Grafana** | [http://localhost:3000](http://localhost:3000) | `admin` / `admin` | 主要可视化中心。 |
+| **Prometheus** | [http://localhost:9090](http://localhost:9090) | - | 指标查询和状态。 |
+| **Jaeger UI** | [http://localhost:16686](http://localhost:16686) | - | 辅助追踪可视化。 |
+| **Tempo** | [http://localhost:3200](http://localhost:3200) | - | Tempo 状态/指标。 |

- Grafana: `http://localhost:3000` (默认账号/密码：`admin`/`admin`)
- Jaeger: `http://localhost:16686`
- Prometheus: `http://localhost:9090`
+## 配置

-## 配置可观测性
+### 数据持久化

-```shell
-export RUSTFS_OBS_ENDPOINT="http://localhost:4317" # OpenTelemetry Collector 地址
+数据存储在以下 Docker 卷中：
+
+-   `prometheus-data`: Prometheus 指标
+-   `tempo-data`: Tempo 追踪 (WAL 和 Blocks)
+-   `loki-data`: Loki 日志 (Chunks 和 Rules)
+-   `jaeger-data`: Jaeger 追踪 (Badger DB)
+
+要清除所有数据：
+
+```bash
+docker compose down -v
 ```
+
+### 自定义
+
+-   **Prometheus**: 编辑 `prometheus.yml` 以添加抓取目标或告警规则。
+-   **Grafana**: 仪表盘和数据源从 `grafana/` 目录预置。
+-   **Collector**: 编辑 `otel-collector-config.yaml` 以修改管道、处理器或导出器。
+
+## 故障排除
+
+-   **服务健康**: 使用 `docker compose ps` 检查服务健康状况。
+-   **日志**: 使用 `docker compose logs -f <service_name>` 查看特定服务的日志。
+-   **Otel Collector**: 检查 `http://localhost:13133` 获取健康状态，检查 `http://localhost:1888/debug/pprof/` 进行性能分析。
@@ -13,164 +13,241 @@
 # limitations under the License.

 services:
-
-  tempo-init:
-    image: busybox:latest
-    command: [ "sh", "-c", "chown -R 10001:10001 /var/tempo" ]
+  rustfs:
+    security_opt:
+      - "no-new-privileges:true"
+    image: rustfs/rustfs:latest
+    container_name: rustfs-server
+    ports:
+      - "9000:9000" # S3 API port
+      - "9001:9001" # Console port
+    environment:
+      - RUSTFS_VOLUMES=/data/rustfs
+      - RUSTFS_ADDRESS=0.0.0.0:9000
+      - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_CORS_ALLOWED_ORIGINS=*
+      - RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS=*
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
+      - RUSTFS_OBS_LOGGER_LEVEL=info
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
    volumes:
-      - ./tempo-data:/var/tempo
-    user: root
+      - rustfs-data:/data/rustfs
    networks:
      - otel-network
-    restart: "no"
+    restart: unless-stopped
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "sh",
+          "-c",
+          "curl -f http://127.0.0.1:9000/health && curl -f http://127.0.0.1:9001/rustfs/console/health",
+        ]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    depends_on:
+      otel-collector:
+        condition: service_started
+
+  rustfs-init:
+    image: alpine
+    container_name: rustfs-init
+    volumes:
+      - rustfs-data:/data
+    networks:
+      - otel-network
+    command: >
+      sh -c "
+        chown -R 10001:10001 /data &&
+        echo 'Volume Permissions fixed' &&
+        exit 0
+      "
+    restart: no
+
+  # --- Tracing ---

  tempo:
    image: grafana/tempo:latest
-    user: "10001" # The container must be started with root to execute chown in the script
-    command: [ "-config.file=/etc/tempo.yaml" ] # This is passed as a parameter to the entry point script
+    container_name: tempo
+    command: ["-config.file=/etc/tempo.yaml"]
    volumes:
      - ./tempo.yaml:/etc/tempo.yaml:ro
-      - ./tempo-data:/var/tempo
+      - tempo-data:/var/tempo
    ports:
      - "3200:3200" # tempo
-      - "24317:4317" # otlp grpc
-      - "24318:4318" # otlp http
-    restart: unless-stopped
+      - "4317"      # otlp grpc
+      - "4318"      # otlp http
    networks:
      - otel-network
+    restart: unless-stopped
+    depends_on:
+      - redpanda
    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/metrics" ]
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3200/ready"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 15s

-  otel-collector:
-    image: otel/opentelemetry-collector-contrib:latest
-    environment:
-      - TZ=Asia/Shanghai
-    volumes:
-      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
+  redpanda:
+    image: redpandadata/redpanda:latest # for tempo ingest
+    container_name: redpanda
    ports:
-      - "1888:1888" # pprof
-      - "8888:8888" # Prometheus metrics for Collector
-      - "8889:8889" # Prometheus metrics for application indicators
-      - "13133:13133" # health check
-      - "4317:4317" # OTLP gRPC
-      - "4318:4318" # OTLP HTTP
-      - "55679:55679" # zpages
+      - "9092:9092"
    networks:
      - otel-network
-    depends_on:
-      jaeger:
-        condition: service_started
-      tempo:
-        condition: service_started
-      prometheus:
-        condition: service_started
-      loki:
-        condition: service_started
+    restart: unless-stopped
+    command: >
+      redpanda start --overprovisioned
+      --mode=dev-container
+      --kafka-addr=PLAINTEXT://0.0.0.0:9092
+      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
+
+  jaeger:
+    image: jaegertracing/jaeger:latest
+    container_name: jaeger
+    environment:
+      - SPAN_STORAGE_TYPE=badger
+      - BADGER_EPHEMERAL=false
+      - BADGER_DIRECTORY_VALUE=/badger/data
+      - BADGER_DIRECTORY_KEY=/badger/key
+      - COLLECTOR_OTLP_ENABLED=true
+    volumes:
+      - ./jaeger.yaml:/etc/jaeger/config.yml
+      - jaeger-data:/badger
+    ports:
+      - "16686:16686" # Web UI
+      - "14269:14269" # Admin/Metrics
+      - "4317" # otlp grpc
+      - "4318" # otlp http
+    command: ["--config", "/etc/jaeger/config.yml"]
+    networks:
+      - otel-network
+    restart: unless-stopped
    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:14269"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
+
+  # --- Metrics ---
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    ports:
+      - "9090:9090"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--web.enable-otlp-receiver" # Enable OTLP
+      - "--web.enable-remote-write-receiver" # Enable remote write
+      - "--enable-feature=promql-experimental-functions" # Enable info()
+      - "--storage.tsdb.retention.time=30d"
+    restart: unless-stopped
+    networks:
+      - otel-network
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"]
      interval: 10s
      timeout: 5s
      retries: 3

-  jaeger:
-    image: jaegertracing/jaeger:latest
-    environment:
-      - TZ=Asia/Shanghai
-      - SPAN_STORAGE_TYPE=memory
-      - COLLECTOR_OTLP_ENABLED=true
-    ports:
-      - "16686:16686" # Web UI
-      - "14317:4317" # OTLP gRPC
-      - "14318:4318" # OTLP HTTP
-      - "18888:8888" # collector
-    networks:
-      - otel-network
-    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:16686" ]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-  prometheus:
-    image: prom/prometheus:latest
-    environment:
-      - TZ=Asia/Shanghai
-    volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - ./prometheus-data:/prometheus
-    ports:
-      - "9090:9090"
-    command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--web.enable-otlp-receiver' # Enable OTLP
-      - '--web.enable-remote-write-receiver' # Enable remote write
-      - '--enable-feature=promql-experimental-functions' # Enable info()
-      - '--storage.tsdb.min-block-duration=15m' # Minimum block duration
-      - '--storage.tsdb.max-block-duration=1h'  # Maximum block duration
-      - '--log.level=info'
-      - '--storage.tsdb.retention.time=30d'
-      - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
-    restart: unless-stopped
-    networks:
-      - otel-network
-    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
-      interval: 10s
-      timeout: 5s
-      retries: 3
+  # --- Logging ---
+
  loki:
    image: grafana/loki:latest
-    environment:
-      - TZ=Asia/Shanghai
+    container_name: loki
    volumes:
-      - ./loki-config.yaml:/etc/loki/local-config.yaml:ro
+      - ./loki.yaml:/etc/loki/loki.yaml:ro
+      - loki-data:/loki
    ports:
      - "3100:3100"
-    command: -config.file=/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/loki.yaml
    networks:
      - otel-network
+    restart: unless-stopped
    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3100/ready"]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+
+  # --- Collection ---
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
+    ports:
+      - "1888:1888"   # pprof
+      - "8888:8888"   # Prometheus metrics for Collector
+      - "8889:8889"   # Prometheus metrics for application indicators
+      - "13133:13133" # health check
+      - "4317:4317"   # OTLP gRPC
+      - "4318:4318"   # OTLP HTTP
+      - "55679:55679" # zpages
+    networks:
+      - otel-network
+    restart: unless-stopped
+    depends_on:
+      - tempo
+      - jaeger
+      - prometheus
+      - loki
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:13133"]
      interval: 10s
      timeout: 5s
      retries: 3
+
+  # --- Visualization ---
+
  grafana:
    image: grafana/grafana:latest
+    container_name: grafana
    ports:
-      - "3000:3000"  # Web UI
-    volumes:
-      - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_SECURITY_ADMIN_USER=admin
-      - TZ=Asia/Shanghai
-      - GF_INSTALL_PLUGINS=grafana-pyroscope-datasource
-    restart: unless-stopped
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/etc/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
    networks:
      - otel-network
+    restart: unless-stopped
    depends_on:
      - prometheus
      - tempo
      - loki
    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"]
      interval: 10s
      timeout: 5s
      retries: 3

 volumes:
-  prometheus-data:
+  rustfs-data:
  tempo-data:
+  jaeger-data:
+  prometheus-data:
+  loki-data:
+  grafana-data:

 networks:
  otel-network:
    driver: bridge
-    name: "network_otel_config"
+    name: "network_otel"
    ipam:
      config:
        - subnet: 172.28.0.0/16
@@ -1,108 +0,0 @@
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    uid: prometheus
-    access: proxy
-    orgId: 1
-    url: http://prometheus:9090
-    basicAuth: false
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-  - name: Tempo
-    type: tempo
-    access: proxy
-    orgId: 1
-    url: http://tempo:3200
-    basicAuth: false
-    isDefault: true
-    version: 1
-    editable: false
-    apiVersion: 1
-    uid: tempo
-    jsonData:
-      httpMethod: GET
-      serviceMap:
-        datasourceUid: prometheus
-      streamingEnabled:
-        search: true
-      tracesToLogsV2:
-        # Field with an internal link pointing to a logs data source in Grafana.
-        # datasourceUid value must match the uid value of the logs data source.
-        datasourceUid: 'loki'
-        spanStartTimeShift: '-1h'
-        spanEndTimeShift: '1h'
-        tags: [ 'job', 'instance', 'pod', 'namespace' ]
-        filterByTraceID: false
-        filterBySpanID: false
-        customQuery: true
-        query: 'method="$${__span.tags.method}"'
-        tracesToMetrics:
-          datasourceUid: 'prometheus'
-          spanStartTimeShift: '-1h'
-          spanEndTimeShift: '1h'
-          tags: [ { key: 'service.name', value: 'service' }, { key: 'job' } ]
-          queries:
-            - name: 'Sample query'
-              query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
-        tracesToProfiles:
-          datasourceUid: 'grafana-pyroscope-datasource'
-          tags: [ 'job', 'instance', 'pod', 'namespace' ]
-          profileTypeId: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds'
-          customQuery: true
-          query: 'method="$${__span.tags.method}"'
-        serviceMap:
-          datasourceUid: 'prometheus'
-        nodeGraph:
-          enabled: true
-        search:
-          hide: false
-        traceQuery:
-          timeShiftEnabled: true
-          spanStartTimeShift: '-1h'
-          spanEndTimeShift: '1h'
-        spanBar:
-          type: 'Tag'
-          tag: 'http.path'
-        streamingEnabled:
-          search: true
-  - name: Jaeger
-    type: jaeger
-    uid: Jaeger
-    url: http://jaeger:16686
-    basicAuth: false
-    access: proxy
-    readOnly: false
-    isDefault: false
-    jsonData:
-      tracesToLogsV2:
-        # Field with an internal link pointing to a logs data source in Grafana.
-        # datasourceUid value must match the uid value of the logs data source.
-        datasourceUid: 'loki'
-        spanStartTimeShift: '1h'
-        spanEndTimeShift: '-1h'
-        tags: [ 'job', 'instance', 'pod', 'namespace' ]
-        filterByTraceID: false
-        filterBySpanID: false
-        customQuery: true
-        query: 'method="$${__span.tags.method}"'
-      tracesToMetrics:
-        datasourceUid: 'Prometheus'
-        spanStartTimeShift: '1h'
-        spanEndTimeShift: '-1h'
-        tags: [ { key: 'service.name', value: 'service' }, { key: 'job' } ]
-        queries:
-          - name: 'Sample query'
-            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
-      nodeGraph:
-        enabled: true
-      traceQuery:
-        timeShiftEnabled: true
-        spanStartTimeShift: '1h'
-        spanEndTimeShift: '-1h'
-      spanBar:
-        type: 'None'
@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: "default"
+    orgId: 1
+    folder: ""
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    options:
+      path: /etc/grafana/dashboards
@@ -0,0 +1,97 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    orgId: 1
+    url: http://prometheus:9090
+    isDefault: true
+    version: 1
+    editable: false
+    jsonData:
+      httpMethod: GET
+      exemplarTraceIdDestinations:
+        - name: trace_id
+          datasourceUid: tempo
+
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    access: proxy
+    orgId: 1
+    url: http://tempo:3200
+    isDefault: false
+    version: 1
+    editable: false
+    jsonData:
+      httpMethod: GET
+      serviceMap:
+        datasourceUid: prometheus
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '1s'
+        spanEndTimeShift: '-1s'
+        filterByTraceID: true
+        filterBySpanID: false
+      tracesToMetrics:
+        datasourceUid: prometheus
+        tags: [ { key: 'service.name' }, { key: 'job' } ]
+        queries:
+          - name: 'Service-Level Latency'
+            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
+          - name: 'Service-Level Calls'
+            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
+          - name: 'Service-Level Errors'
+            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
+      nodeGraph:
+        enabled: true
+
+  - name: Loki
+    type: loki
+    uid: loki
+    orgId: 1
+    url: http://loki:3100
+    isDefault: false
+    version: 1
+    editable: false
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: 'trace_id=(\w+)'
+          name: 'TraceID'
+          url: '$${__value.raw}'
+
+  - name: Jaeger
+    type: jaeger
+    uid: jaeger
+    url: http://jaeger:16686
+    access: proxy
+    isDefault: false
+    editable: false
+    jsonData:
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '1s'
+        spanEndTimeShift: '-1s'
+        filterByTraceID: true
+        filterBySpanID: false
@@ -0,0 +1,78 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    url: http://prometheus:9090
+    access: proxy
+    isDefault: true
+    editable: false
+    jsonData:
+      httpMethod: GET
+      exemplarTraceIdDestinations:
+        - name: trace_id
+          datasourceUid: tempo
+
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    access: proxy
+    url: http://tempo:3200
+    isDefault: false
+    editable: false
+    jsonData:
+      httpMethod: GET
+      serviceMap:
+        datasourceUid: prometheus
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '-1h'
+        spanEndTimeShift: '1h'
+        filterByTraceID: true
+        filterBySpanID: false
+      tracesToMetrics:
+        datasourceUid: prometheus
+        tags: [ { key: 'service.name' }, { key: 'job' } ]
+        queries:
+          - name: 'Service-Level Latency'
+            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
+          - name: 'Service-Level Calls'
+            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
+          - name: 'Service-Level Errors'
+            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
+      nodeGraph:
+        enabled: true
+
+  - name: Loki
+    type: loki
+    uid: loki
+    url: http://loki:3100
+    basicAuth: false
+    isDefault: false
+    editable: false
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: 'trace_id=(\w+)'
+          name: 'TraceID'
+          url: '$${__value.raw}'
+
+  - name: Jaeger
+    type: jaeger
+    uid: jaeger
+    url: http://jaeger:16686
+    access: proxy
+    isDefault: false
+    editable: false
+    jsonData:
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '1s'
+        spanEndTimeShift: '-1s'
+        filterByTraceID: true
+        filterBySpanID: false
@@ -1,114 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-service:
-  extensions: [ jaeger_storage, jaeger_query, remote_sampling, healthcheckv2 ]
-  pipelines:
-    traces:
-      receivers: [ otlp, jaeger, zipkin ]
-      processors: [ batch, adaptive_sampling ]
-      exporters: [ jaeger_storage_exporter ]
-  telemetry:
-    resource:
-      service.name: jaeger
-    metrics:
-      level: detailed
-      readers:
-        - pull:
-            exporter:
-              prometheus:
-                host: 0.0.0.0
-                port: 8888
-    logs:
-      level: debug
-    # TODO Initialize telemetry tracer once OTEL released new feature.
-    # https://github.com/open-telemetry/opentelemetry-collector/issues/10663
-
-extensions:
-  healthcheckv2:
-    use_v2: true
-    http:
-
-  # pprof:
-  #   endpoint: 0.0.0.0:1777
-  # zpages:
-  #   endpoint: 0.0.0.0:55679
-
-  jaeger_query:
-    storage:
-      traces: some_store
-      traces_archive: another_store
-    ui:
-      config_file: ./cmd/jaeger/config-ui.json
-      log_access: true
-    # The maximum duration that is considered for clock skew adjustments.
-    # Defaults to 0 seconds, which means it's disabled.
-    max_clock_skew_adjust: 0s
-    grpc:
-      endpoint: 0.0.0.0:16685
-    http:
-      endpoint: 0.0.0.0:16686
-
-  jaeger_storage:
-    backends:
-      some_store:
-        memory:
-          max_traces: 1000000
-          max_events: 100000
-      another_store:
-        memory:
-          max_traces: 1000000
-    metric_backends:
-      some_metrics_storage:
-        prometheus:
-          endpoint: http://prometheus:9090
-          normalize_calls: true
-          normalize_duration: true
-
-  remote_sampling:
-    # You can either use file or adaptive sampling strategy in remote_sampling
-    # file:
-    #   path: ./cmd/jaeger/sampling-strategies.json
-    adaptive:
-      sampling_store: some_store
-      initial_sampling_probability: 0.1
-    http:
-    grpc:
-
-receivers:
-  otlp:
-    protocols:
-      grpc:
-      http:
-
-  jaeger:
-    protocols:
-      grpc:
-      thrift_binary:
-      thrift_compact:
-      thrift_http:
-
-  zipkin:
-
-processors:
-  batch:
-  metadata_keys: [ "span.kind", "http.method", "http.status_code", "db.system", "db.statement", "messaging.system", "messaging.destination", "messaging.operation","span.events","span.links" ]
-  # Adaptive Sampling Processor is required to support adaptive sampling.
-  # It expects remote_sampling extension with `adaptive:` config to be enabled.
-  adaptive_sampling:
-
-exporters:
-  jaeger_storage_exporter:
-    trace_storage: some_store
-
@@ -0,0 +1,74 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+service:
+  extensions: [jaeger_storage, jaeger_query]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [jaeger_storage_exporter, spanmetrics]
+    metrics/spanmetrics:
+      receivers: [spanmetrics]
+      exporters: [prometheus]
+  telemetry:
+    resource:
+      service.name: jaeger
+    metrics:
+      level: detailed
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: 0.0.0.0
+                port: 8888
+    logs:
+      level: DEBUG
+
+extensions:
+  jaeger_query:
+    storage:
+      traces: some_storage
+      metrics: some_metrics_storage
+  jaeger_storage:
+    backends:
+      some_storage:
+        memory:
+          max_traces: 100000
+    metric_backends:
+      some_metrics_storage:
+        prometheus:
+          endpoint: http://prometheus:9090
+          normalize_calls: true
+          normalize_duration: true
+
+connectors:
+  spanmetrics:
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "0.0.0.0:4317"
+      http:
+        endpoint: "0.0.0.0:4318"
+
+processors:
+  batch:
+
+exporters:
+  jaeger_storage_exporter:
+    trace_storage: some_storage
+  prometheus:
+    endpoint: "0.0.0.0:8889"
@@ -11,22 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 auth_enabled: false

 server:
  http_listen_port: 3100
-  grpc_listen_port: 9096
-  log_level: debug
+  grpc_listen_port: 9095
+  log_level: info
  grpc_server_max_concurrent_streams: 1000

 common:
  instance_addr: 127.0.0.1
-  path_prefix: /tmp/loki
+  path_prefix: /loki
  storage:
    filesystem:
-      chunks_directory: /tmp/loki/chunks
-      rules_directory: /tmp/loki/rules
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    kvstore:
@@ -39,12 +38,6 @@ query_range:
        enabled: true
        max_size_mb: 100

-limits_config:
-  metric_aggregation_enabled: true
-  max_line_size: 256KB
-  max_line_size_truncate: false
-  allow_structured_metadata: true
-
 schema_config:
  configs:
    - from: 2020-10-24
@@ -54,29 +47,17 @@ schema_config:
      index:
        prefix: index_
        period: 24h
-      row_shards: 16
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  allow_structured_metadata: true
+  max_line_size: 256KB

 pattern_ingester:
  enabled: true
  metric_aggregation:
    loki_address: localhost:3100

-ruler:
-  alertmanager_url: http://localhost:9093
-
 frontend:
  encoding: protobuf
-
-
-# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
-# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
-#
-# Statistics help us better understand how Loki is used, and they show us performance
-# levels for most users. This helps us prioritize features and documentation.
-# For more information on what's sent, look at
-# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
-# Refer to the buildReport method to see what goes into a report.
-#
-# If you would like to disable reporting, uncomment the following lines:
-#analytics:
-#  reporting_enabled: false
@@ -15,70 +15,70 @@
 receivers:
  otlp:
    protocols:
-      grpc: # OTLP gRPC receiver
+      grpc:
        endpoint: 0.0.0.0:4317
-      http: # OTLP HTTP receiver
+      http:
        endpoint: 0.0.0.0:4318

 processors:
-  batch: # Batch processor to improve throughput
-    timeout: 5s
-    send_batch_size: 1000
-    metadata_keys: [ ]
-    metadata_cardinality_limit: 1000
+  batch:
+    timeout: 1s
+    send_batch_size: 1024
  memory_limiter:
    check_interval: 1s
-    limit_mib: 512
+    limit_mib: 1024
+    spike_limit_mib: 256
  transform/logs:
    log_statements:
      - context: log
        statements:
-          # Extract Body as attribute "message"
          - set(attributes["message"], body.string)
-          # Retain the original Body
          - set(attributes["log.body"], body.string)

 exporters:
-  otlp/traces: # OTLP exporter for trace data
-    endpoint: "http://jaeger:4317"  # OTLP gRPC endpoint for Jaeger
+  otlp/tempo:
+    endpoint: "tempo:4317"
    tls:
-      insecure: true  # TLS is disabled in the development environment and a certificate needs to be configured in the production environment.
-    compression: gzip  # Enable compression to reduce network bandwidth
+      insecure: true
+    compression: gzip
    retry_on_failure:
-      enabled: true  # Enable retry on failure
-      initial_interval: 1s  # Initial interval for retry
-      max_interval: 30s  # Maximum interval for retry
-      max_elapsed_time: 300s  # Maximum elapsed time for retry
+      enabled: true
+      initial_interval: 1s
+      max_interval: 30s
+      max_elapsed_time: 300s
    sending_queue:
-      enabled: true  # Enable sending queue
-      num_consumers: 10  # Number of consumers
-      queue_size: 5000  # Queue size
-  otlp/tempo: # OTLP exporter for trace data
-    endpoint: "http://tempo:4317"  # OTLP gRPC endpoint for tempo
+      enabled: true
+      num_consumers: 10
+      queue_size: 5000
+
+  otlp/jaeger:
+    endpoint: "jaeger:4317"
    tls:
-      insecure: true  # TLS is disabled in the development environment and a certificate needs to be configured in the production environment.
-    compression: gzip  # Enable compression to reduce network bandwidth
+      insecure: true
+    compression: gzip
    retry_on_failure:
-      enabled: true  # Enable retry on failure
-      initial_interval: 1s  # Initial interval for retry
-      max_interval: 30s  # Maximum interval for retry
-      max_elapsed_time: 300s  # Maximum elapsed time for retry
+      enabled: true
+      initial_interval: 1s
+      max_interval: 30s
+      max_elapsed_time: 300s
    sending_queue:
-      enabled: true  # Enable sending queue
-      num_consumers: 10  # Number of consumers
-      queue_size: 5000  # Queue size
-  prometheus: # Prometheus exporter for metrics data
-    endpoint: "0.0.0.0:8889"  # Prometheus scraping endpoint
-    namespace: "metrics"  # indicator prefix
-    send_timestamps: true  # Send timestamp
-    metric_expiration: 5m  # Metric expiration time
+      enabled: true
+      num_consumers: 10
+      queue_size: 5000
+
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+    send_timestamps: true
+    metric_expiration: 5m
    resource_to_telemetry_conversion:
-      enabled: true  # Enable resource to telemetry conversion
-  otlphttp/loki: # Loki exporter for log data
+      enabled: true
+
+  otlphttp/loki:
    endpoint: "http://loki:3100/otlp"
    tls:
      insecure: true
-    compression: gzip  # Enable compression to reduce network bandwidth
+    compression: gzip
+
 extensions:
  health_check:
    endpoint: 0.0.0.0:13133
@@ -86,13 +86,14 @@ extensions:
    endpoint: 0.0.0.0:1888
  zpages:
    endpoint: 0.0.0.0:55679
+
 service:
-  extensions: [ health_check, pprof, zpages ]  # Enable extension
+  extensions: [ health_check, pprof, zpages ]
  pipelines:
    traces:
      receivers: [ otlp ]
      processors: [ memory_limiter, batch ]
-      exporters: [ otlp/traces, otlp/tempo ]
+      exporters: [ otlp/tempo, otlp/jaeger ]
    metrics:
      receivers: [ otlp ]
      processors: [ batch ]
@@ -103,20 +104,13 @@ service:
      exporters: [ otlphttp/loki ]
  telemetry:
    logs:
-      level: "debug"  # Collector log level
-      encoding: "json"  # Log encoding: console or json
+      level: "info"
+      encoding: "json"
    metrics:
-      level: "detailed" # Can be basic, normal, detailed
+      level: "normal"
      readers:
-        - periodic:
-            exporter:
-              otlp:
-                protocol: http/protobuf
-                endpoint: http://otel-collector:4318
        - pull:
            exporter:
              prometheus:
                host: '0.0.0.0'
                port: 8888
-
-
@@ -1 +0,0 @@
-*
@@ -17,27 +17,40 @@ global:
  evaluation_interval: 15s
  external_labels:
    cluster: 'rustfs-dev'  # Label to identify the cluster
-    relica: '1'  # Replica identifier
+    replica: '1'  # Replica identifier

 scrape_configs:
-  - job_name: 'otel-collector-internal'
+  - job_name: 'otel-collector'
    static_configs:
      - targets: [ 'otel-collector:8888' ]  # Scrape metrics from Collector
    scrape_interval: 10s
+
  - job_name: 'rustfs-app-metrics'
    static_configs:
      - targets: [ 'otel-collector:8889' ]  # Application indicators
    scrape_interval: 15s
    metric_relabel_configs:
+      - source_labels: [ __name__ ]
+        regex: 'go_.*'
+        action: drop  # Drop Go runtime metrics if not needed
+
  - job_name: 'tempo'
    static_configs:
      - targets: [ 'tempo:3200' ]  # Scrape metrics from Tempo
+
  - job_name: 'jaeger'
    static_configs:
-      - targets: [ 'jaeger:8888' ]  # Jaeger admin port
+      - targets: [ 'jaeger:14269' ]  # Jaeger admin port (14269 is standard for admin/metrics)
+
+  - job_name: 'loki'
+    static_configs:
+      - targets: [ 'loki:3100' ]
+
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: [ 'localhost:9090' ]

 otlp:
-  # Recommended attributes to be promoted to labels.
  promote_resource_attributes:
    - service.instance.id
    - service.name
@@ -56,10 +69,8 @@ otlp:
    - k8s.pod.name
    - k8s.replicaset.name
    - k8s.statefulset.name
-  # Ingest OTLP data keeping all characters in metric/label names.
  translation_strategy: NoUTF8EscapingWithSuffixes

 storage:
-  # OTLP is a push-based protocol, Out of order samples is a common scenario.
  tsdb:
-    out_of_order_time_window: 30m
+    out_of_order_time_window: 30m
@@ -1 +0,0 @@
-*
@@ -1,33 +1,55 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+partition_ring_live_store: true
 stream_over_http_enabled: true
+
 server:
  http_listen_port: 3200
  log_level: info

-query_frontend:
-  search:
-    duration_slo: 5s
-    throughput_bytes_slo: 1.073741824e+09
-    metadata_slo:
-      duration_slo: 5s
-      throughput_bytes_slo: 1.073741824e+09
-  trace_by_id:
-    duration_slo: 5s
-
 distributor:
+  ingester_write_path_enabled: false
+  kafka_write_path_enabled: true
  receivers:
    otlp:
      protocols:
        grpc:
-          endpoint: "0.0.0.0:4317"
+          endpoint: "tempo:4317"
        http:
-          endpoint: "0.0.0.0:4318"
+          endpoint: "tempo:4318"
+  #log_received_spans:
+  #    enabled: true
+  #  log_discarded_spans:
+  #    enabled: true

-ingester:
-  max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+backend_scheduler:
+  provider:
+    compaction:
+      compaction:
+        block_retention: 1h

-compactor:
+backend_worker:
+  backend_scheduler_addr: localhost:3200
  compaction:
-    block_retention: 1h # overall Tempo trace retention. set for demo purposes
+    block_retention: 1h
+  ring:
+    kvstore:
+      store: memberlist
+
+querier:
+  query_live_store: true

 metrics_generator:
  registry:
@@ -39,19 +61,34 @@ metrics_generator:
    remote_write:
      - url: http://prometheus:9090/api/v1/write
        send_exemplars: true
-  traces_storage:
-    path: /var/tempo/generator/traces
+
+query_frontend:
+  rf1_after: "1999-01-01T00:00:00Z"
+  mcp_server:
+    enabled: true

 storage:
  trace:
-    backend: local # backend configuration to use
+    backend: local
    wal:
-      path: /var/tempo/wal # where to store the wal locally
+      path: /var/tempo/wal
    local:
      path: /var/tempo/blocks

 overrides:
  defaults:
    metrics_generator:
-      processors: [ service-graphs, span-metrics, local-blocks ] # enables metrics generator
-      generate_native_histograms: both
+      processors: ["span-metrics", "service-graphs", "local-blocks"]
+      generate_native_histograms: both
+
+ingest:
+  enabled: true
+  kafka:
+    address: redpanda:9092
+    topic: tempo-ingest
+
+block_builder:
+  consume_cycle_duration: 30s
+
+usage_report:
+  reporting_enabled: false
@@ -5,71 +5,57 @@

 English | [中文](README_ZH.md)

-This directory contains the configuration files for setting up an observability stack with OpenObserve and OpenTelemetry
-Collector.
+This directory contains the configuration for an **alternative** observability stack using OpenObserve.

-### Overview
+## ⚠️ Note

-This setup provides a complete observability solution for your applications:
+For the **recommended** observability stack (Prometheus, Grafana, Tempo, Loki), please see `../observability/`.

- **OpenObserve**: A modern, open-source observability platform for logs, metrics, and traces.
- **OpenTelemetry Collector**: Collects and processes telemetry data before sending it to OpenObserve.
+## 🌟 Overview

-### Setup Instructions
+OpenObserve is a lightweight, all-in-one observability platform that handles logs, metrics, and traces in a single binary. This setup is ideal for:
+- Resource-constrained environments.
+- Quick setup and testing.
+- Users who prefer a unified UI.

-1. **Prerequisites**:
-    - Docker and Docker Compose installed
-    - Sufficient memory resources (minimum 2GB recommended)
+## 🚀 Quick Start

-2. **Starting the Services**:
-   ```bash
-   cd .docker/openobserve-otel
-   docker compose -f docker-compose.yml  up -d
-   ```
-
-3. **Accessing the Dashboard**:
-    - OpenObserve UI: http://localhost:5080
-    - Default credentials:
-        - Username: root@rustfs.com
-        - Password: rustfs123
-
-### Configuration
-
-#### OpenObserve Configuration
-
-The OpenObserve service is configured with:
-
- Root user credentials
- Data persistence through a volume mount
- Memory cache enabled
- Health checks
- Exposed ports:
-    - 5080: HTTP API and UI
-    - 5081: OTLP gRPC
-
-#### OpenTelemetry Collector Configuration
-
-The collector is configured to:
-
- Receive telemetry data via OTLP (HTTP and gRPC)
- Collect logs from files
- Process data in batches
- Export data to OpenObserve
- Manage memory usage
-
-### Integration with Your Application
-
-To send telemetry data from your application, configure your OpenTelemetry SDK to send data to:
-
- OTLP gRPC: `localhost:4317`
- OTLP HTTP: `localhost:4318`
-
-For example, in a Rust application using the `rustfs-obs` library:
+### 1. Start Services

 ```bash
-export RUSTFS_OBS_ENDPOINT=http://localhost:4317
-export RUSTFS_OBS_SERVICE_NAME=yourservice
-export RUSTFS_OBS_SERVICE_VERSION=1.0.0
-export RUSTFS_OBS_ENVIRONMENT=development
+cd .docker/openobserve-otel
+docker compose up -d
 ```

+### 2. Access Dashboard
+
+- **URL**: [http://localhost:5080](http://localhost:5080)
+- **Username**: `root@rustfs.com`
+- **Password**: `rustfs123`
+
+## 🛠️ Configuration
+
+### OpenObserve
+
+- **Persistence**: Data is persisted to a Docker volume.
+- **Ports**:
+    - `5080`: HTTP API and UI
+    - `5081`: OTLP gRPC
+
+### OpenTelemetry Collector
+
+- **Receivers**: OTLP (gRPC `4317`, HTTP `4318`)
+- **Exporters**: Sends data to OpenObserve.
+
+## 🔗 Integration
+
+Configure your application to send OTLP data to the collector:
+
+- **Endpoint**: `http://localhost:4318` (HTTP) or `localhost:4317` (gRPC)
+
+Example for RustFS:
+
+```bash
+export RUSTFS_OBS_ENDPOINT=http://localhost:4318
+export RUSTFS_OBS_SERVICE_NAME=rustfs-node-1
+```
@@ -5,71 +5,57 @@

 [English](README.md) | 中文

-## 中文
+本目录包含使用 OpenObserve 的**替代**可观测性技术栈配置。

-本目录包含搭建 OpenObserve 和 OpenTelemetry Collector 可观测性栈的配置文件。
+## ⚠️ 注意

-### 概述
+对于**推荐**的可观测性技术栈（Prometheus, Grafana, Tempo, Loki），请参阅 `../observability/`。

-此设置为应用程序提供了完整的可观测性解决方案：
+## 🌟 概览

- **OpenObserve**：现代化、开源的可观测性平台，用于日志、指标和追踪。
- **OpenTelemetry Collector**：收集和处理遥测数据，然后将其发送到 OpenObserve。
+OpenObserve 是一个轻量级、一体化的可观测性平台，在一个二进制文件中处理日志、指标和追踪。此设置非常适合：
+- 资源受限的环境。
+- 快速设置和测试。
+- 喜欢统一 UI 的用户。

-### 设置说明
+## 🚀 快速开始

-1. **前提条件**：
-    - 已安装 Docker 和 Docker Compose
-    - 足够的内存资源（建议至少 2GB）
-
-2. **启动服务**：
-   ```bash
-   cd .docker/openobserve-otel
-   docker compose -f docker-compose.yml  up -d
-   ```
-
-3. **访问仪表板**：
-    - OpenObserve UI：http://localhost:5080
-    - 默认凭据：
-        - 用户名：root@rustfs.com
-        - 密码：rustfs123
-
-### 配置
-
-#### OpenObserve 配置
-
-OpenObserve 服务配置：
-
- 根用户凭据
- 通过卷挂载实现数据持久化
- 启用内存缓存
- 健康检查
- 暴露端口：
-    - 5080：HTTP API 和 UI
-    - 5081：OTLP gRPC
-
-#### OpenTelemetry Collector 配置
-
-收集器配置为：
-
- 通过 OTLP（HTTP 和 gRPC）接收遥测数据
- 从文件中收集日志
- 批处理数据
- 将数据导出到 OpenObserve
- 管理内存使用
-
-### 与应用程序集成
-
-要从应用程序发送遥测数据，将 OpenTelemetry SDK 配置为发送数据到：
-
- OTLP gRPC:`localhost:4317`
- OTLP HTTP:`localhost:4318`
-
-例如，在使用 `rustfs-obs` 库的 Rust 应用程序中：
+### 1. 启动服务

 ```bash
-export RUSTFS_OBS_ENDPOINT=http://localhost:4317
-export RUSTFS_OBS_SERVICE_NAME=yourservice
-export RUSTFS_OBS_SERVICE_VERSION=1.0.0
-export RUSTFS_OBS_ENVIRONMENT=development
-```
+cd .docker/openobserve-otel
+docker compose up -d
+```
+
+### 2. 访问仪表盘
+
+- **URL**: [http://localhost:5080](http://localhost:5080)
+- **用户名**: `root@rustfs.com`
+- **密码**: `rustfs123`
+
+## 🛠️ 配置
+
+### OpenObserve
+
+- **持久化**: 数据持久化到 Docker 卷。
+- **端口**:
+    - `5080`: HTTP API 和 UI
+    - `5081`: OTLP gRPC
+
+### OpenTelemetry Collector
+
+- **接收器**: OTLP (gRPC `4317`, HTTP `4318`)
+- **导出器**: 将数据发送到 OpenObserve。
+
+## 🔗 集成
+
+配置您的应用程序将 OTLP 数据发送到收集器：
+
+- **端点**: `http://localhost:4318` (HTTP) 或 `localhost:4317` (gRPC)
+
+RustFS 示例：
+
+```bash
+export RUSTFS_OBS_ENDPOINT=http://localhost:4318
+export RUSTFS_OBS_SERVICE_NAME=rustfs-node-1
+```
@@ -0,0 +1 @@
+../AGENTS.md
@@ -29,6 +29,8 @@ updates:
    ignore:
      - dependency-name: "object_store"
        versions: [ "0.13.x" ]
+      - dependency-name: "libunftp"
+        versions: [ "0.23.x" ]
    groups:
      s3s:
        update-types:
@@ -161,42 +161,52 @@ jobs:
    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    env:
-      RUSTFLAGS: ${{ matrix.cross == 'false' && '-C target-cpu=native' || '' }}
+      RUSTFLAGS: ${{ matrix.rustflags }}
    strategy:
      fail-fast: false
      matrix:
        include:
          # Linux builds
+          # Use x86-64-v2 (SSE4.2 baseline) instead of native to ensure distributed
+          # binaries run on older x86_64 CPUs (e.g. Intel Celeron/Atom, Synology NAS).
+          # See: https://github.com/rustfs/rustfs/issues/1838
          - os: ubicloud-standard-2
            target: x86_64-unknown-linux-musl
            cross: false
            platform: linux
+            rustflags: ''
          - os: ubicloud-standard-2
            target: aarch64-unknown-linux-musl
            cross: true
            platform: linux
+            rustflags: ''
          - os: ubicloud-standard-2
            target: x86_64-unknown-linux-gnu
            cross: false
            platform: linux
+            rustflags: ''
          - os: ubicloud-standard-2
            target: aarch64-unknown-linux-gnu
            cross: true
            platform: linux
+            rustflags: ''
          # macOS builds
          - os: macos-latest
            target: aarch64-apple-darwin
            cross: false
            platform: macos
+            rustflags: ''
          - os: macos-latest
            target: x86_64-apple-darwin
            cross: false
            platform: macos
+            rustflags: ''
          # Windows builds (temporarily disabled)
          - os: windows-latest
            target: x86_64-pc-windows-msvc
            cross: false
            platform: windows
+            rustflags: ''
          #- os: windows-latest
          #  target: aarch64-pc-windows-msvc
          #  cross: true
@@ -19,7 +19,6 @@ on:
    branches: [ main ]
    paths-ignore:
      - "**.md"
-      - "**.txt"
      - "docs/**"
      - "deploy/**"
      - "scripts/dev_*.sh"
@@ -39,7 +38,6 @@ on:
    branches: [ main ]
    paths-ignore:
      - "**.md"
-      - "**.txt"
      - "docs/**"
      - "deploy/**"
      - "scripts/dev_*.sh"
@@ -132,6 +130,9 @@ jobs:
      - name: Run clippy lints
        run: cargo clippy --all-targets --all-features -- -D warnings

+      - name: Check layered dependencies
+        run: ./scripts/check_layer_dependencies.sh
+
  e2e-tests:
    name: End-to-End Tests
    needs: skip-check
@@ -180,3 +181,43 @@ jobs:
          name: e2e-test-logs-${{ github.run_number }}
          path: /tmp/rustfs.log
          retention-days: 3
+
+  s3-implemented-tests:
+    name: S3 Implemented Tests
+    needs: skip-check
+    if: needs.skip-check.outputs.should_skip != 'true'
+    runs-on: ubicloud-standard-4
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-s3tests-${{ hashFiles('**/Cargo.lock') }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build debug binary
+        run: |
+          touch rustfs/build.rs
+          cargo build -p rustfs --bins --jobs 2
+
+      - name: Run implemented s3-tests
+        run: |
+          DEPLOY_MODE=binary \
+          RUSTFS_BINARY=./target/debug/rustfs \
+          TEST_MODE=single \
+          MAXFAIL=1 \
+          ./scripts/s3-tests/run.sh
+
+      - name: Upload s3 test artifacts
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: s3tests-implemented-${{ github.run_number }}
+          path: artifacts/s3tests-single/**
+          if-no-files-found: ignore
+          retention-days: 3
@@ -266,6 +266,17 @@ jobs:
    if: needs.build-check.outputs.should_build == 'true'
    runs-on: ubicloud-standard-2
    timeout-minutes: 60
+    strategy:
+          fail-fast: false
+          matrix:
+            include:
+              - variant: musl
+                file: Dockerfile
+                suffix: ""
+              - variant: glibc
+                file: Dockerfile.glibc
+                suffix: "-glibc"
+
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
@@ -296,6 +307,7 @@ jobs:
          VERSION="${{ needs.build-check.outputs.version }}"
          SHORT_SHA="${{ needs.build-check.outputs.short_sha }}"
          CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}"
+          VARIANT_SUFFIX="${{ matrix.suffix }}"

          # Convert version format for Dockerfile compatibility
          case "$VERSION" in
@@ -326,14 +338,14 @@ jobs:

          # Generate tags based on build type
          # Only support release and prerelease builds (no development builds)
-          TAGS="${{ env.REGISTRY_DOCKERHUB }}:${VERSION}"
+          TAGS="${{ env.REGISTRY_DOCKERHUB }}:${VERSION}${VARIANT_SUFFIX}"

          # Add channel tags for prereleases and latest for stable
          if [[ "$CREATE_LATEST" == "true" ]]; then
            # TODO: Temporary change - the current alpha version will also create the latest tag
            # After the version is stabilized, the logic here remains unchanged, but the upstream CREATE_LATEST setting needs to be restored.
            # Stable release (and temporary alpha versions)
-            TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest"
+            TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX}"
          elif [[ "$BUILD_TYPE" == "prerelease" ]]; then
            # Prerelease channel tags (alpha, beta, rc)
            if [[ "$VERSION" == *"alpha"* ]]; then
@@ -345,7 +357,7 @@ jobs:
            fi

            if [[ -n "$CHANNEL" ]]; then
-              TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}"
+              TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}${VARIANT_SUFFIX}"
            fi
          fi

@@ -372,15 +384,15 @@ jobs:
        uses: docker/build-push-action@v6
        with:
          context: .
-          file: Dockerfile
+          file: ${{ matrix.file }}
          platforms: ${{ env.DOCKER_PLATFORMS }}
          push: ${{ needs.build-check.outputs.should_push == 'true' }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: |
-            type=gha,scope=docker-binary
+            type=gha,scope=docker-${{ matrix.variant }}
          cache-to: |
-            type=gha,mode=max,scope=docker-binary
+            type=gha,mode=max,scope=docker-${{ matrix.variant }}
          build-args: |
            BUILDTIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
            VERSION=${{ needs.build-check.outputs.version }}
@@ -0,0 +1,61 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Update Nix Flake
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 0' # Weekly on Sundays
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  update-flake:
+    name: Update flake.lock
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Install Nix
+        uses: DeterminateSystems/determinate-nix-action@v3
+
+      - name: Check Nix flake inputs
+        uses: DeterminateSystems/flake-checker-action@v12
+
+      - name: Update flake.lock
+        id: update
+        uses: DeterminateSystems/update-flake-lock@main
+        with:
+          pr-title: "chore(deps): update flake.lock"
+          pr-labels: |
+            dependencies
+            nix
+            automated
+          commit-msg: "chore(deps): update flake.lock"
+          pr-reviewers: houseme, heihutu
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Log PR details
+        if: steps.update.outputs.pull-request-number
+        run: |
+          echo "Pull Request created: ${{ steps.update.outputs.pull-request-number }}"
@@ -0,0 +1,84 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Nix CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'flake.nix'
+      - 'flake.lock'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
+      - '.github/workflows/nix.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'flake.nix'
+      - 'flake.lock'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
+      - '.github/workflows/nix.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+permissions:
+  contents: read
+
+jobs:
+  nix-validation:
+    name: Nix Build & Check
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: Setup Magic Nix Cache
+        uses: DeterminateSystems/magic-nix-cache-action@v13
+        
+      - name: Setup Flake Checker
+        uses: DeterminateSystems/flake-checker-action@v12
+
+      - name: Verify Flake
+        run: |
+          echo "Checking flake structure and evaluation..."
+          nix flake show
+          nix flake check --print-build-logs
+
+      - name: Build RustFS
+        run: |
+          echo "Building the default package..."
+          nix build .#default --print-build-logs
+
+      - name: Test Binary
+        run: |
+          echo "Verifying the built binary..."
+          if [ -x "./result/bin/rustfs" ]; then
+              ./result/bin/rustfs --help
+              echo "Binary verification successful."
+          else
+              echo "Error: Binary not found or not executable at ./result/bin/rustfs"
+              exit 1
+          fi
@@ -2,6 +2,7 @@
 .DS_Store
 .idea
 .vscode
+.cursor
 .direnv/
 /test
 /logs
@@ -39,3 +40,8 @@ artifacts/
 PR_DESCRIPTION.md
 IMPLEMENTATION_PLAN.md
 scripts/s3-tests/selected_tests.txt
+docs
+
+# nix stuff
+result*
+*.gz
@@ -1,702 +0,0 @@
-# RustFS Project AI Coding Rules
-
-## 🚨🚨🚨 CRITICAL DEVELOPMENT RULES - ZERO TOLERANCE 🚨🚨🚨
-
-### ⛔️ ABSOLUTE PROHIBITION: NEVER COMMIT DIRECTLY TO MASTER/MAIN BRANCH ⛔️
-
-**🔥 THIS IS THE MOST CRITICAL RULE - VIOLATION WILL RESULT IN IMMEDIATE REVERSAL 🔥**
-
- **🚫 ZERO DIRECT COMMITS TO MAIN/MASTER BRANCH - ABSOLUTELY FORBIDDEN**
- **🚫 ANY DIRECT COMMIT TO MAIN BRANCH MUST BE IMMEDIATELY REVERTED**
- **🚫 NO EXCEPTIONS FOR HOTFIXES, EMERGENCIES, OR URGENT CHANGES**
- **🚫 NO EXCEPTIONS FOR SMALL CHANGES, TYPOS, OR DOCUMENTATION UPDATES**
- **🚫 NO EXCEPTIONS FOR ANYONE - MAINTAINERS, CONTRIBUTORS, OR ADMINS**
-
-### 📋 MANDATORY WORKFLOW - STRICTLY ENFORCED
-
-**EVERY SINGLE CHANGE MUST FOLLOW THIS WORKFLOW:**
-
-1. **Check current branch**: `git branch` (MUST NOT be on main/master)
-2. **Switch to main**: `git checkout main`
-3. **Pull latest**: `git pull origin main`
-4. **Create feature branch**: `git checkout -b feat/your-feature-name`
-5. **Make changes ONLY on feature branch**
-6. **Test thoroughly before committing**
-7. **Commit and push to feature branch**: `git push origin feat/your-feature-name`
-8. **Create Pull Request**: Use `gh pr create` (MANDATORY)
-9. **Wait for PR approval**: NO self-merging allowed
-10. **Merge through GitHub interface**: ONLY after approval
-
-### 🔒 ENFORCEMENT MECHANISMS
-
- **Branch protection rules**: Main branch is protected
- **Pre-commit hooks**: Will block direct commits to main
- **CI/CD checks**: All PRs must pass before merging
- **Code review requirement**: At least one approval needed
- **Automated reversal**: Direct commits to main will be automatically reverted
-
-## 🎯 Core AI Development Principles
-
-### Five Execution Steps
-
-#### 1. Task Analysis and Planning
- **Clear Objectives**: Deeply understand task requirements and expected results before starting coding
- **Plan Development**: List specific files, components, and functions that need modification, explaining the reasons for changes
- **Risk Assessment**: Evaluate the impact of changes on existing functionality, develop rollback plans
-
-#### 2. Precise Code Location
- **File Identification**: Determine specific files and line numbers that need modification
- **Impact Analysis**: Avoid modifying irrelevant files, clearly state the reason for each file modification
- **Minimization Principle**: Unless explicitly required by the task, do not create new abstraction layers or refactor existing code
-
-#### 3. Minimal Code Changes
- **Focus on Core**: Only write code directly required by the task
- **Avoid Redundancy**: Do not add unnecessary logs, comments, tests, or error handling
- **Isolation**: Ensure new code does not interfere with existing functionality, maintain code independence
-
-#### 4. Strict Code Review
- **Correctness Check**: Verify the correctness and completeness of code logic
- **Style Consistency**: Ensure code conforms to established project coding style
- **Side Effect Assessment**: Evaluate the impact of changes on downstream systems
-
-#### 5. Clear Delivery Documentation
- **Change Summary**: Detailed explanation of all modifications and reasons
- **File List**: List all modified files and their specific changes
- **Risk Statement**: Mark any assumptions or potential risk points
-
-### Core Principles
- **🎯 Precise Execution**: Strictly follow task requirements, no arbitrary innovation
- **⚡ Efficient Development**: Avoid over-design, only do necessary work
- **🛡️ Safe and Reliable**: Always follow development processes, ensure code quality and system stability
- **🔒 Cautious Modification**: Only modify when clearly knowing what needs to be changed and having confidence
-
-### Additional AI Behavior Rules
-
-1. **Use English for all code comments and documentation** - All comments, variable names, function names, documentation, and user-facing text in code should be in English
-2. **Clean up temporary scripts after use** - Any temporary scripts, test files, or helper files created during AI work should be removed after task completion
-3. **Only make confident modifications** - Do not make speculative changes or "convenient" modifications outside the task scope. If uncertain about a change, ask for clarification rather than guessing
-
-## Project Overview
-
-RustFS is a high-performance distributed object storage system written in Rust, compatible with S3 API. The project adopts a modular architecture, supporting erasure coding storage, multi-tenant management, observability, and other enterprise-level features.
-
-## Core Architecture Principles
-
-### 1. Modular Design
-
- Project uses Cargo workspace structure, containing multiple independent crates
- Core modules: `rustfs` (main service), `ecstore` (erasure coding storage), `common` (shared components)
- Functional modules: `iam` (identity management), `madmin` (management interface), `crypto` (encryption), etc.
- Tool modules: `cli` (command line tool), `crates/*` (utility libraries)
-
-### 2. Asynchronous Programming Pattern
-
- Comprehensive use of `tokio` async runtime
- Prioritize `async/await` syntax
- Use `async-trait` for async methods in traits
- Avoid blocking operations, use `spawn_blocking` when necessary
-
-### 3. Error Handling Strategy
-
- **Use modular, type-safe error handling with `thiserror`**
- Each module should define its own error type using `thiserror::Error` derive macro
- Support error chains and context information through `#[from]` and `#[source]` attributes
- Use `Result<T>` type aliases for consistency within each module
- Error conversion between modules should use explicit `From` implementations
- Follow the pattern: `pub type Result<T> = core::result::Result<T, Error>`
- Use `#[error("description")]` attributes for clear error messages
- Support error downcasting when needed through `other()` helper methods
- Implement `Clone` for errors when required by the domain logic
-
-## Code Style Guidelines
-
-### 1. Formatting Configuration
-
-```toml
-max_width = 130
-fn_call_width = 90
-single_line_let_else_max_width = 100
-```
-
-### 2. **🔧 MANDATORY Code Formatting Rules**
-
-**CRITICAL**: All code must be properly formatted before committing. This project enforces strict formatting standards to maintain code consistency and readability.
-
-#### Pre-commit Requirements (MANDATORY)
-
-Before every commit, you **MUST**:
-
-1. **Format your code**:
-   ```bash
-   cargo fmt --all
-   ```
-
-2. **Verify formatting**:
-   ```bash
-   cargo fmt --all --check
-   ```
-
-3. **Pass clippy checks**:
-   ```bash
-   cargo clippy --all-targets --all-features -- -D warnings
-   ```
-
-4. **Ensure compilation**:
-   ```bash
-   cargo check --all-targets
-   ```
-
-#### Quick Commands
-
-Use these convenient Makefile targets for common tasks:
-
-```bash
-# Format all code
-make fmt
-
-# Check if code is properly formatted
-make fmt-check
-
-# Run clippy checks
-make clippy
-
-# Run compilation check
-make check
-
-# Run tests
-make test
-
-# Run all pre-commit checks (format + clippy + check + test)
-make pre-commit
-
-# Setup git hooks (one-time setup)
-make setup-hooks
-```
-
-### 3. Naming Conventions
-
- Use `snake_case` for functions, variables, modules
- Use `PascalCase` for types, traits, enums
- Constants use `SCREAMING_SNAKE_CASE`
- Global variables prefix `GLOBAL_`, e.g., `GLOBAL_Endpoints`
- Use meaningful and descriptive names for variables, functions, and methods
- Avoid meaningless names like `temp`, `data`, `foo`, `bar`, `test123`
- Choose names that clearly express the purpose and intent
-
-### 4. Type Declaration Guidelines
-
- **Prefer type inference over explicit type declarations** when the type is obvious from context
- Let the Rust compiler infer types whenever possible to reduce verbosity and improve maintainability
- Only specify types explicitly when:
-  - The type cannot be inferred by the compiler
-  - Explicit typing improves code clarity and readability
-  - Required for API boundaries (function signatures, public struct fields)
-  - Needed to resolve ambiguity between multiple possible types
-
-### 5. Documentation Comments
-
- Public APIs must have documentation comments
- Use `///` for documentation comments
- Complex functions add `# Examples` and `# Parameters` descriptions
- Error cases use `# Errors` descriptions
- Always use English for all comments and documentation
- Avoid meaningless comments like "debug 111" or placeholder text
-
-### 6. Import Guidelines
-
- Standard library imports first
- Third-party crate imports in the middle
- Project internal imports last
- Group `use` statements with blank lines between groups
-
-## Asynchronous Programming Guidelines
-
-### 1. Trait Definition
-
-```rust
-#[async_trait::async_trait]
-pub trait StorageAPI: Send + Sync {
-    async fn get_object(&self, bucket: &str, object: &str) -> Result<ObjectInfo>;
-}
-```
-
-### 2. Error Handling
-
-```rust
-// Use ? operator to propagate errors
-async fn example_function() -> Result<()> {
-    let data = read_file("path").await?;
-    process_data(data).await?;
-    Ok(())
-}
-```
-
-### 3. Concurrency Control
-
- Use `Arc` and `Mutex`/`RwLock` for shared state management
- Prioritize async locks from `tokio::sync`
- Avoid holding locks for long periods
-
-## Logging and Tracing Guidelines
-
-### 1. Tracing Usage
-
-```rust
-#[tracing::instrument(skip(self, data))]
-async fn process_data(&self, data: &[u8]) -> Result<()> {
-    info!("Processing {} bytes", data.len());
-    // Implementation logic
-}
-```
-
-### 2. Log Levels
-
- `error!`: System errors requiring immediate attention
- `warn!`: Warning information that may affect functionality
- `info!`: Important business information
- `debug!`: Debug information for development use
- `trace!`: Detailed execution paths
-
-### 3. Structured Logging
-
-```rust
-info!(
-    counter.rustfs_api_requests_total = 1_u64,
-    key_request_method = %request.method(),
-    key_request_uri_path = %request.uri().path(),
-    "API request processed"
-);
-```
-
-## Error Handling Guidelines
-
-### 1. Error Type Definition
-
-```rust
-// Use thiserror for module-specific error types
-#[derive(thiserror::Error, Debug)]
-pub enum MyError {
-    #[error("IO error: {0}")]
-    Io(#[from] std::io::Error),
-
-    #[error("Storage error: {0}")]
-    Storage(#[from] ecstore::error::StorageError),
-
-    #[error("Custom error: {message}")]
-    Custom { message: String },
-
-    #[error("File not found: {path}")]
-    FileNotFound { path: String },
-
-    #[error("Invalid configuration: {0}")]
-    InvalidConfig(String),
-}
-
-// Provide Result type alias for the module
-pub type Result<T> = core::result::Result<T, MyError>;
-```
-
-### 2. Error Helper Methods
-
-```rust
-impl MyError {
-    /// Create error from any compatible error type
-    pub fn other<E>(error: E) -> Self
-    where
-        E: Into<Box<dyn std::error::Error + Send + Sync>>,
-    {
-        MyError::Io(std::io::Error::other(error))
-    }
-}
-```
-
-### 3. Error Context and Propagation
-
-```rust
-// Use ? operator for clean error propagation
-async fn example_function() -> Result<()> {
-    let data = read_file("path").await?;
-    process_data(data).await?;
-    Ok(())
-}
-
-// Add context to errors
-fn process_with_context(path: &str) -> Result<()> {
-    std::fs::read(path)
-        .map_err(|e| MyError::Custom {
-            message: format!("Failed to read {}: {}", path, e)
-        })?;
-    Ok(())
-}
-```
-
-## Performance Optimization Guidelines
-
-### 1. Memory Management
-
- Use `Bytes` instead of `Vec<u8>` for zero-copy operations
- Avoid unnecessary cloning, use reference passing
- Use `Arc` for sharing large objects
-
-### 2. Concurrency Optimization
-
-```rust
-// Use join_all for concurrent operations
-let futures = disks.iter().map(|disk| disk.operation());
-let results = join_all(futures).await;
-```
-
-### 3. Caching Strategy
-
- Use `LazyLock` for global caching
- Implement LRU cache to avoid memory leaks
-
-## Testing Guidelines
-
-### 1. Unit Tests
-
-```rust
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use test_case::test_case;
-
-    #[tokio::test]
-    async fn test_async_function() {
-        let result = async_function().await;
-        assert!(result.is_ok());
-    }
-
-    #[test_case("input1", "expected1")]
-    #[test_case("input2", "expected2")]
-    fn test_with_cases(input: &str, expected: &str) {
-        assert_eq!(function(input), expected);
-    }
-}
-```
-
-### 2. Integration Tests
-
- Use `e2e_test` module for end-to-end testing
- Simulate real storage environments
-
-### 3. Test Quality Standards
-
- Write meaningful test cases that verify actual functionality
- Avoid placeholder or debug content like "debug 111", "test test", etc.
- Use descriptive test names that clearly indicate what is being tested
- Each test should have a clear purpose and verify specific behavior
- Test data should be realistic and representative of actual use cases
-
-## Cross-Platform Compatibility Guidelines
-
-### 1. CPU Architecture Compatibility
-
- **Always consider multi-platform and different CPU architecture compatibility** when writing code
- Support major architectures: x86_64, aarch64 (ARM64), and other target platforms
- Use conditional compilation for architecture-specific code:
-
-```rust
-#[cfg(target_arch = "x86_64")]
-fn optimized_x86_64_function() { /* x86_64 specific implementation */ }
-
-#[cfg(target_arch = "aarch64")]
-fn optimized_aarch64_function() { /* ARM64 specific implementation */ }
-
-#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
-fn generic_function() { /* Generic fallback implementation */ }
-```
-
-### 2. Platform-Specific Dependencies
-
- Use feature flags for platform-specific dependencies
- Provide fallback implementations for unsupported platforms
- Test on multiple architectures in CI/CD pipeline
-
-### 3. Endianness Considerations
-
- Use explicit byte order conversion when dealing with binary data
- Prefer `to_le_bytes()`, `from_le_bytes()` for consistent little-endian format
- Use `byteorder` crate for complex binary format handling
-
-### 4. SIMD and Performance Optimizations
-
- Use portable SIMD libraries like `wide` or `packed_simd`
- Provide fallback implementations for non-SIMD architectures
- Use runtime feature detection when appropriate
-
-## Security Guidelines
-
-### 1. Memory Safety
-
- Disable `unsafe` code (workspace.lints.rust.unsafe_code = "deny")
- Use `rustls` instead of `openssl`
-
-### 2. Authentication and Authorization
-
-```rust
-// Use IAM system for permission checks
-let identity = iam.authenticate(&access_key, &secret_key).await?;
-iam.authorize(&identity, &action, &resource).await?;
-```
-
-## Configuration Management Guidelines
-
-### 1. Environment Variables
-
- Use `RUSTFS_` prefix
- Support both configuration files and environment variables
- Provide reasonable default values
-
-### 2. Configuration Structure
-
-```rust
-#[derive(Debug, Deserialize, Clone)]
-pub struct Config {
-    pub address: String,
-    pub volumes: String,
-    #[serde(default)]
-    pub console_enable: bool,
-}
-```
-
-## Dependency Management Guidelines
-
-### 1. Workspace Dependencies
-
- Manage versions uniformly at workspace level
- Use `workspace = true` to inherit configuration
-
-### 2. Feature Flags
-
-```rust
-[features]
-default = ["file"]
-gpu = ["dep:nvml-wrapper"]
-kafka = ["dep:rdkafka"]
-```
-
-## Deployment and Operations Guidelines
-
-### 1. Containerization
-
- Provide Dockerfile and docker-compose configuration
- Support multi-stage builds to optimize image size
-
-### 2. Observability
-
- Integrate OpenTelemetry for distributed tracing
- Support Prometheus metrics collection
- Provide Grafana dashboards
-
-### 3. Health Checks
-
-```rust
-// Implement health check endpoint
-async fn health_check() -> Result<HealthStatus> {
-    // Check component status
-}
-```
-
-## Code Review Checklist
-
-### 1. **Code Formatting and Quality (MANDATORY)**
-
- [ ] **Code is properly formatted** (`cargo fmt --all --check` passes)
- [ ] **All clippy warnings are resolved** (`cargo clippy --all-targets --all-features -- -D warnings` passes)
- [ ] **Code compiles successfully** (`cargo check --all-targets` passes)
- [ ] **Pre-commit hooks are working** and all checks pass
- [ ] **No formatting-related changes** mixed with functional changes (separate commits)
-
-### 2. Functionality
-
- [ ] Are all error cases properly handled?
- [ ] Is there appropriate logging?
- [ ] Is there necessary test coverage?
-
-### 3. Performance
-
- [ ] Are unnecessary memory allocations avoided?
- [ ] Are async operations used correctly?
- [ ] Are there potential deadlock risks?
-
-### 4. Security
-
- [ ] Are input parameters properly validated?
- [ ] Are there appropriate permission checks?
- [ ] Is information leakage avoided?
-
-### 5. Cross-Platform Compatibility
-
- [ ] Does the code work on different CPU architectures (x86_64, aarch64)?
- [ ] Are platform-specific features properly gated with conditional compilation?
- [ ] Is byte order handling correct for binary data?
- [ ] Are there appropriate fallback implementations for unsupported platforms?
-
-### 6. Code Commits and Documentation
-
- [ ] Does it comply with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)?
- [ ] Are commit messages concise and under 72 characters for the title line?
- [ ] Commit titles should be concise and in English, avoid Chinese
- [ ] Is PR description provided in copyable markdown format for easy copying?
-
-## Common Patterns and Best Practices
-
-### 1. Resource Management
-
-```rust
-// Use RAII pattern for resource management
-pub struct ResourceGuard {
-    resource: Resource,
-}
-
-impl Drop for ResourceGuard {
-    fn drop(&mut self) {
-        // Clean up resources
-    }
-}
-```
-
-### 2. Dependency Injection
-
-```rust
-// Use dependency injection pattern
-pub struct Service {
-    config: Arc<Config>,
-    storage: Arc<dyn StorageAPI>,
-}
-```
-
-### 3. Graceful Shutdown
-
-```rust
-// Implement graceful shutdown
-async fn shutdown_gracefully(shutdown_rx: &mut Receiver<()>) {
-    tokio::select! {
-        _ = shutdown_rx.recv() => {
-            info!("Received shutdown signal");
-            // Perform cleanup operations
-        }
-        _ = tokio::time::sleep(SHUTDOWN_TIMEOUT) => {
-            warn!("Shutdown timeout reached");
-        }
-    }
-}
-```
-
-## Domain-Specific Guidelines
-
-### 1. Storage Operations
-
- All storage operations must support erasure coding
- Implement read/write quorum mechanisms
- Support data integrity verification
-
-### 2. Network Communication
-
- Use gRPC for internal service communication
- HTTP/HTTPS support for S3-compatible API
- Implement connection pooling and retry mechanisms
-
-### 3. Metadata Management
-
- Use FlatBuffers for serialization
- Support version control and migration
- Implement metadata caching
-
-## Branch Management and Development Workflow
-
-### Branch Management
-
- **🚨 CRITICAL: NEVER modify code directly on main or master branch - THIS IS ABSOLUTELY FORBIDDEN 🚨**
- **⚠️ ANY DIRECT COMMITS TO MASTER/MAIN WILL BE REJECTED AND MUST BE REVERTED IMMEDIATELY ⚠️**
- **🔒 ALL CHANGES MUST GO THROUGH PULL REQUESTS - NO DIRECT COMMITS TO MAIN UNDER ANY CIRCUMSTANCES 🔒**
- **Always work on feature branches - NO EXCEPTIONS**
- Always check the .rules.md file before starting to ensure you understand the project guidelines
- **MANDATORY workflow for ALL changes:**
-   1. `git checkout main` (switch to main branch)
-   2. `git pull` (get latest changes)
-   3. `git checkout -b feat/your-feature-name` (create and switch to feature branch)
-   4. Make your changes ONLY on the feature branch
-   5. Test thoroughly before committing
-   6. Commit and push to the feature branch
-   7. **Create a pull request for code review - THIS IS THE ONLY WAY TO MERGE TO MAIN**
-   8. **Wait for PR approval before merging - NEVER merge your own PRs without review**
- Use descriptive branch names following the pattern: `feat/feature-name`, `fix/issue-name`, `refactor/component-name`, etc.
- **Double-check current branch before ANY commit: `git branch` to ensure you're NOT on main/master**
- **Pull Request Requirements:**
-  - All changes must be submitted via PR regardless of size or urgency
-  - PRs must include comprehensive description and testing information
-  - PRs must pass all CI/CD checks before merging
-  - PRs require at least one approval from code reviewers
-  - Even hotfixes and emergency changes must go through PR process
- **Enforcement:**
-  - Main branch should be protected with branch protection rules
-  - Direct pushes to main should be blocked by repository settings
-  - Any accidental direct commits to main must be immediately reverted via PR
-
-### Development Workflow
-
-## 🎯 **Core Development Principles**
-
- **🔴 Every change must be precise - don't modify unless you're confident**
-  - Carefully analyze code logic and ensure complete understanding before making changes
-  - When uncertain, prefer asking users or consulting documentation over blind modifications
-  - Use small iterative steps, modify only necessary parts at a time
-  - Evaluate impact scope before changes to ensure no new issues are introduced
-
- **🚀 GitHub PR creation prioritizes gh command usage**
-  - Prefer using `gh pr create` command to create Pull Requests
-  - Avoid having users manually create PRs through web interface
-  - Provide clear and professional PR titles and descriptions
-  - Using `gh` commands ensures better integration and automation
-
-## 📝 **Code Quality Requirements**
-
- Use English for all code comments, documentation, and variable names
- Write meaningful and descriptive names for variables, functions, and methods
- Avoid meaningless test content like "debug 111" or placeholder values
- Before each change, carefully read the existing code to ensure you understand the code structure and implementation, do not break existing logic implementation, do not introduce new issues
- Ensure each change provides sufficient test cases to guarantee code correctness
- Do not arbitrarily modify numbers and constants in test cases, carefully analyze their meaning to ensure test case correctness
- When writing or modifying tests, check existing test cases to ensure they have scientific naming and rigorous logic testing, if not compliant, modify test cases to ensure scientific and rigorous testing
- **Before committing any changes, run `cargo clippy --all-targets --all-features -- -D warnings` to ensure all code passes Clippy checks**
- After each development completion, first git add . then git commit -m "feat: feature description" or "fix: issue description", ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
- **Keep commit messages concise and under 72 characters** for the title line, use body for detailed explanations if needed
- After each development completion, first git push to remote repository
- After each change completion, summarize the changes, do not create summary files, provide a brief change description, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
- Provide change descriptions needed for PR in the conversation, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
- **Always provide PR descriptions in English** after completing any changes, including:
-  - Clear and concise title following Conventional Commits format
-  - Detailed description of what was changed and why
-  - List of key changes and improvements
-  - Any breaking changes or migration notes if applicable
-  - Testing information and verification steps
- **Provide PR descriptions in copyable markdown format** enclosed in code blocks for easy one-click copying
-
-## 🚫 AI Documentation Generation Restrictions
-
-### Forbidden Summary Documents
-
- **Strictly forbidden to create any form of AI-generated summary documents**
- **Do not create documents containing large amounts of emoji, detailed formatting tables and typical AI style**
- **Do not generate the following types of documents in the project:**
-  - Benchmark summary documents (BENCHMARK*.md)
-  - Implementation comparison analysis documents (IMPLEMENTATION_COMPARISON*.md)
-  - Performance analysis report documents
-  - Architecture summary documents
-  - Feature comparison documents
-  - Any documents with large amounts of emoji and formatted content
- **If documentation is needed, only create when explicitly requested by the user, and maintain a concise and practical style**
- **Documentation should focus on actually needed information, avoiding excessive formatting and decorative content**
- **Any discovered AI-generated summary documents should be immediately deleted**
-
-### Allowed Documentation Types
-
- README.md (project introduction, keep concise)
- Technical documentation (only create when explicitly needed)
- User manual (only create when explicitly needed)
- API documentation (generated from code)
- Changelog (CHANGELOG.md)
-
-These rules should serve as guiding principles when developing the RustFS project, ensuring code quality, performance, and maintainability.
@@ -99,7 +99,17 @@
            "name": "Debug executable target/debug/rustfs",
            "type":  "lldb",
            "request": "launch",
-            "program": "${workspaceFolder}/target/debug/rustfs",
+            "cargo": {
+                "args": [
+                    "run",
+                    "--bin",
+                    "rustfs",
+                    "-j",
+                    "1",
+                    "--profile",
+                    "dev"
+                ]
+            },
            "args": [],
            "cwd": "${workspaceFolder}",
            //"stopAtEntry": false,
@@ -107,7 +117,7 @@
            "env": {
                "RUSTFS_ACCESS_KEY": "rustfsadmin",
                "RUSTFS_SECRET_KEY": "rustfsadmin",
-                "RUSTFS_VOLUMES": "./target/volume/test{1...4}",
+                //"RUSTFS_VOLUMES": "./target/volume/test{1...4}",
                "RUSTFS_ADDRESS": ":9000",
                "RUSTFS_CONSOLE_ENABLE": "true",
                // "RUSTFS_OBS_TRACE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", // jeager otlp http endpoint
@@ -116,11 +126,31 @@
                // "RUSTFS_COMPRESS_ENABLE": "true",
                "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
                "RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
+                "RUST_LOG":"rustfs=debug,ecstore=debug,s3s=debug,iam=debug",
            },
            "sourceLanguages": [
                "rust"
            ],
        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug test_lifecycle_transition_basic",
+            "cargo": {
+                "args": [
+                    "test",
+                    "-p",
+                    "rustfs-scanner",
+                    "--test",
+                    "lifecycle_integration_test",
+                    "serial_tests::test_lifecycle_transition_basic",
+                    "-j",
+                    "1"
+                ]
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        },
        {
            "name": "Debug executable target/debug/test",
            "type":  "lldb",
@@ -140,6 +170,81 @@
            "sourceLanguages": [
                "rust"
            ],
-        }
+        },
+        {
+            "name": "Debug executable target/debug/rustfs with sse",
+            "type":  "lldb",
+            "request": "launch",
+            "program": "${workspaceFolder}/target/debug/rustfs",
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            //"stopAtEntry": false,
+            //"preLaunchTask": "cargo build",
+            "env": {
+                "RUSTFS_ACCESS_KEY": "rustfsadmin",
+                "RUSTFS_SECRET_KEY": "rustfsadmin",
+                "RUSTFS_VOLUMES": "./target/volumes/test{1...4}",
+                "RUSTFS_ADDRESS": ":9000",
+                "RUSTFS_CONSOLE_ENABLE": "true",
+                "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
+                "RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
+                // "RUSTFS_OBS_TRACE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", // jeager otlp http endpoint
+                // "RUSTFS_OBS_METRIC_ENDPOINT": "http://127.0.0.1:4318/v1/metrics", // default otlp http endpoint
+                // "RUSTFS_OBS_LOG_ENDPOINT": "http://127.0.0.1:4318/v1/logs", // default otlp http endpoint
+                // "RUSTFS_COMPRESS_ENABLE": "true",
+
+                // 1. simple sse test key (no kms system)
+                // "__RUSTFS_SSE_SIMPLE_CMK": "2dfNXGHlsEflGVCxb+5DIdGEl1sIvtwX+QfmYasi5QM=",
+
+                // 2. kms local backend test key
+                "RUSTFS_KMS_ENABLE": "true",
+                "RUSTFS_KMS_BACKEND": "local",
+                "RUSTFS_KMS_KEY_DIR": "./target/kms-key-dir",
+                "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
+                "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
+
+                // 3. kms vault backend test key
+                // "RUSTFS_KMS_ENABLE": "true",
+                // "RUSTFS_KMS_BACKEND": "vault",
+                // "RUSTFS_KMS_VAULT_ADDRESS": "http://127.0.0.1:8200",
+                // "RUSTFS_KMS_VAULT_TOKEN": "Dev Token",
+                // "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
+
+            },
+            "sourceLanguages": [
+                "rust"
+            ],
+        },
+        {
+            "name": "E2E test executable target/debug/rustfs",
+            "type":  "lldb",
+            "request": "launch",
+            "program": "${workspaceFolder}/target/debug/rustfs",
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            //"stopAtEntry": false,
+            //"preLaunchTask": "cargo build",
+            "env": {
+                "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=debug",
+                "RUST_BACKTRACE": "full",
+                "RUSTFS_ACCESS_KEY": "rustfsadmin",
+                "RUSTFS_SECRET_KEY": "rustfsadmin",
+                "RUSTFS_VOLUMES": "./target/e2e-test/test{1...4}",
+                "RUSTFS_REGION": "us-east-1",
+                "RUSTFS_ADDRESS": ":9000",
+                "RUSTFS_CONSOLE_ENABLE": "true",
+                "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
+                "RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
+
+                "RUSTFS_KMS_ENABLE": "true",
+                "RUSTFS_KMS_BACKEND": "local",
+                "RUSTFS_KMS_KEY_DIR": "./target/e2e-key-dir",
+                "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
+                "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
+            },
+            "sourceLanguages": [
+                "rust"
+            ],
+        },
    ]
 }
@@ -18,8 +18,8 @@ Or simply run `make pre-commit` which covers all checks. **DO NOT commit if any

 ## Communication Rules

- Respond to the user in Chinese; use English in all other contexts.
- Code and documentation must be written in English only. Chinese text is allowed solely as test data/fixtures when a case explicitly requires Chinese-language content for validation.
+- Respond to the user in the same language used by the user.
+- Code and documentation must be written in English only.
 - **Pull Request titles and descriptions must be written in English** to ensure consistency and accessibility for all contributors.

 ## Project Structure
@@ -71,8 +71,10 @@ Co-locate unit tests with their modules and give behavior-led names. Integration

 ## Environment Variables

- `RUSTFS_ENABLE_SCANNER` - Enable/disable background data scanner (default: true)
- `RUSTFS_ENABLE_HEAL` - Enable/disable auto-heal functionality (default: true)
+- Global configuration environment variables must use flat `RUSTFS_*` names (no module segments), such as `RUSTFS_REGION`, `RUSTFS_ADDRESS`, `RUSTFS_VOLUMES`, and `RUSTFS_LICENSE`.
+- `RUSTFS_SCANNER_ENABLED` - Enable/disable background data scanner (default: true)
+- `RUSTFS_HEAL_ENABLED` - Enable/disable auto-heal functionality (default: true)
+- Deprecated aliases (for pre-beta compatibility) are documented in [the config module README](crates/config/README.md#environment-variable-naming-conventions) and must log warnings when used.
 - For KMS tests: `NO_PROXY=127.0.0.1,localhost` and clear proxy environment variables

 ## Commit & Pull Request Guidelines
@@ -0,0 +1,76 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- **OpenStack Keystone Authentication Integration**: Full support for OpenStack Keystone authentication via X-Auth-Token headers
+  - Tower-based middleware (`KeystoneAuthLayer`) self-contained within `rustfs-keystone` crate
+  - Task-local storage for async-safe credential passing between middleware and auth handlers
+  - Automatic detection of Keystone credentials (access keys prefixed with `keystone:`)
+  - Role-based permission mapping (admin/reseller_admin roles grant owner permissions)
+  - Token caching for high-performance validation with configurable cache size and TTL
+  - Dual authentication support: Keystone and standard AWS Signature v4 work simultaneously
+  - Immediate 401 response for invalid tokens (no fallback to local auth)
+  - XML-formatted error responses compatible with S3 API
+  - Comprehensive integration documentation with manual testing guide
+  - **32 unit and integration tests** covering middleware, auth handlers, task-local storage, and role detection
+
+### Changed
+- **HTTP Server Stack**: Integrated `KeystoneAuthLayer` middleware from `rustfs-keystone` crate into service stack (positioned after ReadinessGateLayer)
+- **IAMAuth**: Enhanced `get_secret_key()` to return empty secret for Keystone credentials (bypasses signature validation)
+- **Auth Module**: Modified `check_key_valid()` to retrieve Keystone credentials from task-local storage and determine admin status
+
+### Technical Details
+- Middleware is self-contained in `rustfs-keystone` crate following the trusted-proxies pattern for integration-specific middleware
+- Uses `BoxBody` pattern for Hyper 1.x compatibility
+- Task-local storage provides request-scoped credential passing without modifying HTTP request/response types
+- Integration preserves existing S3 authentication flow while adding Keystone support
+- Zero breaking changes to existing functionality
+- No new top-level directories in main binary crate (middleware lives in integration crate)
+
+### Documentation
+- Updated `crates/keystone/README.md` with complete integration architecture and workflow
+- Added detailed manual testing guide with 10 test scenarios
+- Updated main `README.md` to list Keystone authentication as available feature
+- Added troubleshooting section for common integration issues
+
+### Configuration
+New environment variables:
+- `RUSTFS_KEYSTONE_ENABLE` - Enable/disable Keystone authentication (default: false)
+- `RUSTFS_KEYSTONE_AUTH_URL` - Keystone API endpoint URL
+- `RUSTFS_KEYSTONE_VERSION` - Keystone API version (v3)
+- `RUSTFS_KEYSTONE_ADMIN_USER` - Admin username for privileged operations
+- `RUSTFS_KEYSTONE_ADMIN_PASSWORD` - Admin password
+- `RUSTFS_KEYSTONE_ADMIN_PROJECT` - Admin project name
+- `RUSTFS_KEYSTONE_ADMIN_DOMAIN` - Admin domain name (default: Default)
+- `RUSTFS_KEYSTONE_CACHE_SIZE` - Token cache size (default: 10000)
+- `RUSTFS_KEYSTONE_CACHE_TTL` - Token cache TTL in seconds (default: 300)
+- `RUSTFS_KEYSTONE_VERIFY_SSL` - Verify SSL certificates (default: true)
+
+### Files Modified
+- `crates/keystone/src/middleware.rs` - Created Keystone authentication middleware (self-contained in keystone crate)
+- `crates/keystone/src/lib.rs` - Exported middleware module and KEYSTONE_CREDENTIALS
+- `crates/keystone/Cargo.toml` - Added Tower/HTTP dependencies for middleware functionality
+- `rustfs/src/server/http.rs` - Integrated KeystoneAuthLayer from rustfs-keystone crate
+- `rustfs/src/auth.rs` - Enhanced IAMAuth and check_key_valid for Keystone support, imported KEYSTONE_CREDENTIALS from rustfs-keystone
+- `crates/keystone/README.md` - Comprehensive integration documentation
+- `README.md` - Added Keystone as available feature
+
+### Testing
+- 16 unit tests in rustfs-keystone crate (config, auth, middleware, identity)
+- 10 integration tests in rustfs-keystone crate (task-local storage, middleware layer, scope isolation)
+- 6 auth unit tests in rustfs crate (role detection, task-local storage, Keystone credential handling)
+- **Total: 32 tests** passing with zero compilation errors
+- Manual testing guide provided for end-to-end validation
+- All tests passing with `cargo test --all --exclude e2e_test`
+
+---
+
+## Previous Releases
+
+See [GitHub Releases](https://github.com/rustfs/rustfs/releases) for previous version history.
@@ -1,85 +1,60 @@
-RustFS Individual Contributor License Agreement
+# RustFS Individual Contributor License Agreement

-Thank you for your interest in contributing documentation and related software code to a project hosted or managed by
-RustFS. In order to clarify the intellectual property license granted with Contributions from any person or entity,
-RustFS must have a Contributor License Agreement ("CLA") on file that has been signed by each Contributor, indicating
-agreement to the license terms below. This version of the Contributor License Agreement allows an individual to submit
-Contributions to the applicable project. If you are making a submission on behalf of a legal entity, then you should
-sign the separate Corporate Contributor License Agreement.
+Thank you for your interest in contributing to RustFS. In order to clarify the intellectual property license granted with Contributions from any person or entity, RustFS, Inc. ("RustFS") must have a Contributor License Agreement ("CLA") on file that has been signed by each Contributor, indicating agreement to the license terms below. This license is for your protection as a Contributor as well as the protection of RustFS and its users; it does not change your rights to use your own Contributions for any other purpose.

-You accept and agree to the following terms and conditions for Your present and future Contributions submitted to
-RustFS. You hereby irrevocably assign and transfer to RustFS all right, title, and interest in and to Your
-Contributions, including all copyrights and other intellectual property rights therein.
+You accept and agree to the following terms and conditions for Your present and future Contributions submitted to RustFS. Except for the license granted herein to RustFS and recipients of software distributed by RustFS, You reserve all right, title, and interest in and to Your Contributions.

-Definitions
+**You understand and agree that You will not receive any royalty or other compensation for Your Contributions.**

-“You” (or “Your”) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this
-Agreement with RustFS. For legal entities, the entity making a Contribution and all other entities that control, are
-controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes
-of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such
-entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares,
-or (iii) beneficial ownership of such entity.
+## 1. Definitions

-“Contribution” shall mean any original work of authorship, including any modifications or additions to an existing work,
-that is intentionally submitted by You to RustFS for inclusion in, or documentation of, any of the products or projects
-owned or managed by RustFS (the "Work"), including without limitation any Work described in Schedule A. For the purposes
-of this definition, "submitted" means any form of electronic or written communication sent to RustFS or its
-representatives, including but not limited to communication on electronic mailing lists, source code control systems,
-and issue tracking systems that are managed by, or on behalf of, RustFS for the purpose of discussing and improving the
-Work.
+*   **"You"** (or **"Your"**) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with RustFS. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.

-Assignment of Copyright
+*   **"Contribution"** shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to RustFS for inclusion in, or documentation of, any of the products or projects owned or managed by RustFS (the "Work"). For the purposes of this definition, "submitted" means any form of electronic or written communication sent to RustFS or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, RustFS for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution."

-Subject to the terms and conditions of this Agreement, You hereby irrevocably assign and transfer to RustFS all right,
-title, and interest in and to Your Contributions, including all copyrights and other intellectual property rights
-therein, for the entire term of such rights, including all renewals and extensions. You agree to execute all documents
-and take all actions as may be reasonably necessary to vest in RustFS the ownership of Your Contributions and to assist
-RustFS in perfecting, maintaining, and enforcing its rights in Your Contributions.
+## 2. Grant of Copyright License

-Grant of Patent License
+Subject to the terms and conditions of this Agreement, You hereby grant to RustFS and to recipients of software distributed by RustFS a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works under any license, including proprietary or commercial licenses and open-source licenses, at RustFS's sole discretion.

-Subject to the terms and conditions of this Agreement, You hereby grant to RustFS and to recipients of documentation and
-software distributed by RustFS a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as
-stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the
-Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your
-Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was
-submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or
-counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes
-direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for
-that Contribution or Work shall terminate as of the date such litigation is filed.
+## 3. Grant of Patent License

-You represent that you are legally entitled to grant the above assignment and license.
+Subject to the terms and conditions of this Agreement, You hereby grant to RustFS and to recipients of software distributed by RustFS a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.

-You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of
-others). You represent that Your Contribution submissions include complete details of any third-party license or other
-restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which
-are associated with any part of Your Contributions.
+## 4. Representations

-You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You
-may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You
-provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
-including, without limitation, any warranties or conditions of TITLE, NON- INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR
-A PARTICULAR PURPOSE.
+You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to RustFS, or that your employer has executed a separate Corporate CLA with RustFS.

-Should You wish to submit work that is not Your original creation, You may submit it to RustFS separately from any
-Contribution, identifying the complete details of its source and of any license or other restriction (including, but not
-limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously
-marking the work as "Submitted on behalf of a third-party: [named here]”.
+You represent that each of Your Contributions is Your original creation. You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which are associated with any part of Your Contributions.

-You agree to notify RustFS of any facts or circumstances of which you become aware that would make these representations
-inaccurate in any respect.
+## 5. Support and Warranty

-Modification of CLA
+You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of **TITLE**, **NON-INFRINGEMENT**, **MERCHANTABILITY**, or **FITNESS FOR A PARTICULAR PURPOSE**.

-RustFS reserves the right to update or modify this CLA in the future. Any updates or modifications to this CLA shall
-apply only to Contributions made after the effective date of the revised CLA. Contributions made prior to the update
-shall remain governed by the version of the CLA that was in effect at the time of submission. It is not necessary for
-all Contributors to re-sign the CLA when the CLA is updated or modified.
+## 6. Third-Party Work

-Governing Law and Dispute Resolution
+Should You wish to submit work that is not Your original creation, You may submit it to RustFS separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as "Submitted on behalf of a third-party: [named here]".

-This Agreement will be governed by and construed in accordance with the laws of the People's Republic of China excluding
-that body of laws known as conflict of laws. The parties expressly agree that the United Nations Convention on Contracts
-for the International Sale of Goods will not apply. Any legal action or proceeding arising under this Agreement will be
-brought exclusively in the courts located in Beijing, China, and the parties hereby irrevocably consent to the personal
-jurisdiction and venue therein.
+## 7. Governing Law and Jurisdiction
+
+This Agreement shall be governed by and construed in accordance with the laws of the State of Delaware, United States of America, without regard to its conflict of laws principles. The parties expressly agree that the United Nations Convention on Contracts for the International Sale of Goods will not apply. Any legal action or proceeding arising under or in connection with this Agreement shall be brought exclusively in the state or federal courts located in the State of Delaware, United States of America, and the parties hereby irrevocably consent to the personal jurisdiction and venue therein.
+
+## 8. Severability
+
+If any provision of this Agreement is found to be invalid or unenforceable, the remaining provisions will continue in full force and effect.
+
+## 9. Entire Agreement and Assignment
+
+This Agreement constitutes the entire agreement between the parties with respect to the subject matter hereof and supersedes all prior agreements, understandings, negotiations, and discussions, whether written or oral. RustFS may assign its rights and obligations under this Agreement to any third party. You may not assign Your rights or obligations under this Agreement without the prior written consent of RustFS.
+
+---
+
+**Please read the terms of this Agreement carefully. By submitting a Contribution to RustFS, You agree to be bound by the terms and conditions of this Agreement.**
+
+| | |
+|---|---|
+| **Full Name** | __________________________________ |
+| **GitHub Username** | __________________________________ |
+| **Email Address** | __________________________________ |
+| **Date** | __________________________________ |
+
+*(Electronic signature or acknowledgement via GitHub commit/Pull Request constitutes valid acceptance of this Agreement).*
@@ -17,6 +17,7 @@ members = [
    "rustfs", # Core file system implementation
    "crates/appauth", # Application authentication and authorization
    "crates/audit", # Audit target management system with multi-target fan-out
+    "crates/checksums", # client checksums
    "crates/common", # Shared utilities and data structures
    "crates/config", # Configuration management
    "crates/credentials", # Credential management system
@@ -24,26 +25,29 @@ members = [
    "crates/ecstore", # Erasure coding storage implementation
    "crates/e2e_test", # End-to-end test suite
    "crates/filemeta", # File metadata management
+    "crates/heal", # Erasure set and object healing
    "crates/iam", # Identity and Access Management
+    "crates/keystone", # OpenStack Keystone integration
+    "crates/kms", # Key Management Service
    "crates/lock", # Distributed locking implementation
    "crates/madmin", # Management dashboard and admin API interface
+    "crates/mcp", # MCP server for S3 operations
+    "crates/metrics", # Metrics collection and reporting
    "crates/notify", # Notification system for events
    "crates/obs", # Observability utilities
    "crates/policy", # Policy management
+    "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
    "crates/protos", # Protocol buffer definitions
    "crates/rio", # Rust I/O utilities and abstractions
-    "crates/targets", # Target-specific configurations and utilities
    "crates/s3select-api", # S3 Select API interface
    "crates/s3select-query", # S3 Select query engine
    "crates/scanner", # Scanner for data integrity checks and health monitoring
    "crates/signer", # client signer
-    "crates/checksums", # client checksums
+    "crates/targets", # Target-specific configurations and utilities
+    "crates/trusted-proxies", # Trusted proxies management
    "crates/utils", # Utility functions and helpers
    "crates/workers", # Worker thread pools and task scheduling
    "crates/zip", # ZIP file handling and compression
-    "crates/heal", # Erasure set and object healing
-    "crates/mcp", # MCP server for S3 operations
-    "crates/kms", # Key Management Service
 ]
 resolver = "2"

@@ -51,12 +55,13 @@ resolver = "2"
 edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
-rust-version = "1.90"
+rust-version = "1.93.0"
 version = "0.0.5"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
 categories = ["web-programming", "development-tools", "filesystem", "network-programming"]
+authors = ["RustFS Team"]

 [workspace.lints.rust]
 unsafe_code = "deny"
@@ -78,10 +83,12 @@ rustfs-crypto = { path = "crates/crypto", version = "0.0.5" }
 rustfs-ecstore = { path = "crates/ecstore", version = "0.0.5" }
 rustfs-filemeta = { path = "crates/filemeta", version = "0.0.5" }
 rustfs-iam = { path = "crates/iam", version = "0.0.5" }
+rustfs-keystone = { path = "crates/keystone", version = "0.0.5" }
 rustfs-kms = { path = "crates/kms", version = "0.0.5" }
 rustfs-lock = { path = "crates/lock", version = "0.0.5" }
 rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
 rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
+rustfs-metrics = { path = "crates/metrics", version = "0.0.5" }
 rustfs-notify = { path = "crates/notify", version = "0.0.5" }
 rustfs-obs = { path = "crates/obs", version = "0.0.5" }
 rustfs-policy = { path = "crates/policy", version = "0.0.5" }
@@ -91,154 +98,156 @@ rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
 rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.5" }
 rustfs-scanner = { path = "crates/scanner", version = "0.0.5" }
 rustfs-signer = { path = "crates/signer", version = "0.0.5" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "0.0.5" }
 rustfs-targets = { path = "crates/targets", version = "0.0.5" }
 rustfs-utils = { path = "crates/utils", version = "0.0.5" }
 rustfs-workers = { path = "crates/workers", version = "0.0.5" }
 rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
+rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }

 # Async Runtime and Networking
 async-channel = "2.5.0"
-async-compression = { version = "0.4.37" }
+async-compression = { version = "0.4.41" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
 axum = "0.8.8"
-axum-server = { version = "0.8.0", features = ["tls-rustls"], default-features = false }
-futures = "0.3.31"
-futures-core = "0.3.31"
-futures-util = "0.3.31"
+futures = "0.3.32"
+futures-core = "0.3.32"
+futures-util = "0.3.32"
 pollster = "0.4.0"
 hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
 hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
-hyper-util = { version = "0.1.19", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
+hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
 http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
-reqwest = { version = "0.13.1", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
 socket2 = { version = "0.6.2", features = ["all"] }
 tokio = { version = "1.49.0", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
 tokio-util = { version = "0.7.18", features = ["io", "compat"] }
-tonic = { version = "0.14.3", features = ["gzip"] }
-tonic-prost = { version = "0.14.3" }
-tonic-prost-build = { version = "0.14.3" }
+tonic = { version = "0.14.5", features = ["gzip"] }
+tonic-prost = { version = "0.14.5" }
+tonic-prost-build = { version = "0.14.5" }
 tower = { version = "0.5.3", features = ["timeout"] }
 tower-http = { version = "0.6.8", features = ["cors"] }

 # Serialization and Data Formats
-bytes = { version = "1.11.0", features = ["serde"] }
+bytes = { version = "1.11.1", features = ["serde"] }
 bytesize = "2.3.1"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.3"
-quick-xml = "0.39.0"
-rmcp = { version = "0.14.0" }
+quick-xml = "0.39.2"
+rmcp = { version = "0.17.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.149", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
-schemars = "1.2.0"
+schemars = "1.2.1"

 # Cryptography and Security
-aes-gcm = { version = "0.11.0-rc.2", features = ["rand_core"] }
-argon2 = { version = "0.6.0-rc.6" }
+aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
+argon2 = { version = "0.6.0-rc.7" }
 blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
-chacha20poly1305 = { version = "0.11.0-rc.2" }
+chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
-hmac = { version = "0.13.0-rc.3" }
+hmac = { version = "0.13.0-rc.5" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
-pbkdf2 = "0.13.0-rc.8"
-rsa = { version = "0.10.0-rc.12" }
-rustls = { version = "0.23.36", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
-rustls-pemfile = "2.2.0"
+openidconnect = { version = "4.0", default-features = false }
+pbkdf2 = "0.13.0-rc.9"
+rsa = { version = "0.10.0-rc.16" }
+rustls = { version = "0.23.37", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types = "1.14.0"
-sha1 = "0.11.0-rc.3"
-sha2 = "0.11.0-rc.3"
+sha1 = "0.11.0-rc.5"
+sha2 = "0.11.0-rc.5"
 subtle = "2.6"
 zeroize = { version = "1.8.2", features = ["derive"] }

 # Time and Date
-chrono = { version = "0.4.43", features = ["serde"] }
+chrono = { version = "0.4.44", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.18", features = ["serde"] }
-time = { version = "0.3.46", features = ["std", "parsing", "formatting", "macros", "serde"] }
+jiff = { version = "0.2.22", features = ["serde"] }
+time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }

 # Utilities and Tools
-anyhow = "1.0.100"
-arc-swap = "1.8.0"
+anyhow = "1.0.102"
+arc-swap = "1.8.2"
 astral-tokio-tar = "0.5.6"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.12" }
-aws-credential-types = { version = "1.2.11" }
-aws-sdk-s3 = { version = "1.121.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
-aws-smithy-types = { version = "1.4.1" }
+aws-config = { version = "1.8.14" }
+aws-credential-types = { version = "1.2.13" }
+aws-sdk-s3 = { version = "1.124.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-smithy-http-client = { version = "1.1.11", default-features = false, features = ["default-client", "rustls-aws-lc"] }
+aws-smithy-types = { version = "1.4.5" }
+backtrace = "0.3.76"
 base64 = "0.22.1"
 base64-simd = "0.8.0"
 brotli = "8.0.2"
 cfg-if = "1.0.4"
-clap = { version = "4.5.56", features = ["derive", "env"] }
-const-str = { version = "1.0.0", features = ["std", "proc"] }
-convert_case = "0.10.0"
+clap = { version = "4.5.60", features = ["derive", "env"] }
+const-str = { version = "1.1.0", features = ["std", "proc"] }
+convert_case = "0.11.0"
 criterion = { version = "0.8", features = ["html_reports"] }
 crossbeam-queue = "0.3.12"
-datafusion = "52.1.0"
+datafusion = "52.2.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
-flate2 = "1.1.8"
-flexi_logger = { version = "0.31.8", features = ["trc", "dont_minimize_extra_stacks", "compress", "kv", "json"] }
+flate2 = "1.1.9"
 glob = "0.3.3"
-google-cloud-storage = "1.7.0"
-google-cloud-auth = "1.5.0"
+google-cloud-storage = "1.8.0"
+google-cloud-auth = "1.6.0"
 hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
 hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.180"
+libc = "0.2.182"
 libsystemd = "0.7.2"
-local-ip-address = "0.6.9"
+local-ip-address = "0.6.10"
 lz4 = "1.28.1"
 matchit = "0.9.1"
-md-5 = "0.11.0-rc.3"
+md-5 = "0.11.0-rc.5"
 md5 = "0.8.0"
 mime_guess = "2.0.5"
-moka = { version = "0.12.13", features = ["future"] }
+moka = { version = "0.12.14", features = ["future"] }
 netif = "0.1.6"
-nu-ansi-term = "0.50.3"
 num_cpus = { version = "1.17.0" }
-nvml-wrapper = "0.11.0"
+nvml-wrapper = "0.12.0"
 object_store = "0.12.5"
 parking_lot = "0.12.5"
 path-absolutize = "3.1.1"
 path-clean = "1.0.1"
-pin-project-lite = "0.2.16"
+pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
-rand = { version = "0.10.0-rc.6", features = ["serde"] }
+rand = { version = "0.10.0", features = ["serde"] }
+ratelimit = "0.10.0"
 rayon = "1.11.0"
 reed-solomon-simd = { version = "3.1.0" }
-regex = { version = "1.12.2" }
+regex = { version = "1.12.3" }
 rumqttc = { version = "0.25.1" }
-rustix = { version = "1.1.3", features = ["fs"] }
+rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.1" }
-s3s = { version = "0.13.0-alpha.2", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", rev = "3cdb3fe22fe8a1b7fc3f71ead4beacac2683ba7f" }
-serial_test = "3.3.1"
+s3s = { version = "0.13.0", features = ["minio"] }
+serial_test = "3.4.0"
 shadow-rs = { version = "1.7.0", default-features = false }
 siphasher = "1.0.2"
 smallvec = { version = "1.15.1", features = ["serde"] }
 smartstring = "1.0.1"
 snafu = "0.8.9"
 snap = "1.1.1"
-starshard = { version = "0.6.0", features = ["rayon", "async", "serde"] }
-strum = { version = "0.27.2", features = ["derive"] }
-sysinfo = "0.38.0"
+starshard = { version = "1.1.0", features = ["rayon", "async", "serde"] }
+strum = { version = "0.28.0", features = ["derive"] }
+sysinfo = "0.38.3"
 temp-env = "0.3.6"
-tempfile = "3.24.0"
+tempfile = "3.26.0"
 test-case = "3.3.1"
 thiserror = "2.0.18"
 tracing = { version = "0.1.44" }
@@ -249,13 +258,13 @@ tracing-subscriber = { version = "0.3.22", features = ["env-filter", "time"] }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.20.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.21.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
 vaultrs = { version = "0.7.4" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "7.2.0"
+zip = "8.1.0"
 zstd = "0.13.3"

 # Observability and Metrics
@@ -268,11 +277,9 @@ opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_
 opentelemetry-stdout = { version = "0.31.0" }

 # FTP and SFTP
-libunftp = { version = "0.21.0" }
-russh = { version = "0.57.0", features = ["aws-lc-rs", "rsa"], default-features = false }
-russh-sftp = { version = "2.1.1" }
-ssh-key = { version = "0.7.0-rc.6", features = ["std", "rsa", "ed25519"] }
-suppaftp = { version = "7.1.0", features = ["tokio", "tokio-rustls", "rustls"] }
+libunftp = { version = "0.23.0", features = ["experimental"] }
+unftp-core = "0.1.0"
+suppaftp = { version = "8.0.2", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.7"

 # Performance Analysis and Memory Profiling
@@ -282,7 +289,7 @@ tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unpref
 # Used to control and obtain statistics for jemalloc at runtime
 tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats", "profiling"] }
 # Used to generate pprof-compatible memory profiling data and support symbolization and flame graphs
-jemalloc_pprof = { version = "0.8.1", features = ["symbolize", "flamegraph"] }
+jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
 # Used to generate CPU performance analysis data and flame diagrams
 pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] }

@@ -1,3 +1,17 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 FROM alpine:3.23 AS build

 ARG TARGETARCH
@@ -0,0 +1,115 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:24.04 AS build
+
+ARG TARGETARCH
+ARG RELEASE=latest
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+
+RUN set -eux; \
+    case "$TARGETARCH" in \
+      amd64)  ARCH_SUBSTR="x86_64-gnu"  ;; \
+      arm64)  ARCH_SUBSTR="aarch64-gnu" ;; \
+      *) echo "Unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;; \
+    esac; \
+    \
+    if [ "$RELEASE" = "latest" ]; then \
+      TAG="$(curl -fsSL https://api.github.com/repos/rustfs/rustfs/releases \
+              | grep -o '"tag_name": "[^"]*"' | cut -d'"' -f4 | head -n 1)"; \
+    else \
+      TAG="$RELEASE"; \
+    fi; \
+    \
+    URL="$(curl -fsSL "https://api.github.com/repos/rustfs/rustfs/releases/tags/$TAG" \
+           | grep -o "\"browser_download_url\": \"[^\"]*${ARCH_SUBSTR}[^\"]*\\.zip\"" \
+           | cut -d'"' -f4 | head -n 1)"; \
+    \
+    if [ -z "$URL" ]; then echo "Failed to locate release asset for $ARCH_SUBSTR at tag $TAG" >&2; exit 1; fi; \
+    \
+    curl -fL "$URL" -o rustfs.zip; \
+    unzip -q rustfs.zip -d /build; \
+    \
+    if [ ! -x /build/rustfs ]; then \
+      BIN_PATH="$(unzip -Z -1 rustfs.zip | grep -E '(^|/)rustfs$' | head -n 1 || true)"; \
+      if [ -n "$BIN_PATH" ]; then \
+        mkdir -p /build/.tmp && unzip -q rustfs.zip "$BIN_PATH" -d /build/.tmp && \
+        mv "/build/.tmp/$BIN_PATH" /build/rustfs; \
+      fi; \
+    fi; \
+    [ -x /build/rustfs ] || { echo "rustfs binary not found in asset" >&2; exit 1; }; \
+    chmod +x /build/rustfs; \
+    rm -rf rustfs.zip /build/.tmp || true
+
+FROM ubuntu:24.04
+
+ARG RELEASE=latest
+ARG BUILD_DATE
+ARG VCS_REF
+
+LABEL name="RustFS" \
+      vendor="RustFS Team" \
+      maintainer="RustFS Team <dev@rustfs.com>" \
+      version="v${RELEASE#v}" \
+      release="${RELEASE}" \
+      build-date="${BUILD_DATE}" \
+      vcs-ref="${VCS_REF}" \
+      summary="High-performance distributed object storage system (glibc)" \
+      url="https://rustfs.com" \
+      license="Apache-2.0"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=build /build/rustfs /usr/bin/rustfs
+COPY entrypoint.sh /entrypoint.sh
+
+RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+
+RUN groupadd -g 10001 rustfs && \
+    useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
+    mkdir -p /data /logs && \
+    chown -R rustfs:rustfs /data /logs && \
+    chmod 0750 /data /logs
+
+ENV RUSTFS_ADDRESS=":9000" \
+    RUSTFS_CONSOLE_ADDRESS=":9001" \
+    RUSTFS_ACCESS_KEY="rustfsadmin" \
+    RUSTFS_SECRET_KEY="rustfsadmin" \
+    RUSTFS_CONSOLE_ENABLE="true" \
+    RUSTFS_CORS_ALLOWED_ORIGINS="*" \
+    RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS="*" \
+    RUSTFS_VOLUMES="/data" \
+    RUSTFS_OBS_LOGGER_LEVEL=warn \
+    RUSTFS_OBS_LOG_DIRECTORY=/logs \
+    RUSTFS_OBS_ENVIRONMENT=production
+    
+EXPOSE 9000 9001
+
+VOLUME ["/data"]
+
+USER rustfs
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD ["rustfs"]
@@ -1,4 +1,18 @@
 # syntax=docker/dockerfile:1.6
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Multi-stage Dockerfile for RustFS - LOCAL DEVELOPMENT ONLY
 #
 # IMPORTANT: This Dockerfile builds RustFS from source for local development and testing.
@@ -1,4 +1,4 @@
-[![RustFS](https://github.com/user-attachments/assets/1b5afcd6-a2c3-47ff-8bc3-ce882b0ddca7)](https://rustfs.com)
+[![RustFS](https://github.com/user-attachments/assets/3ba82e75-2f2d-4415-a4aa-1e4ffe9f22fd)](https://rustfs.com)

 <p align="center">RustFS is a high-performance, distributed object storage system built in Rust.</p>

@@ -42,6 +42,7 @@ Unlike other storage systems, RustFS is released under the permissible Apache 2.
 - **High Performance**: Built with Rust to ensure maximum speed and resource efficiency.
 - **Distributed Architecture**: Scalable and fault-tolerant design suitable for large-scale deployments.
 - **S3 Compatibility**: Seamless integration with existing S3-compatible applications and tools.
+- **OpenStack Keystone Integration**: Native support for OpenStack Keystone authentication with X-Auth-Token headers.
 - **Data Lake Support**: Optimized for high-throughput big data and AI workloads.
 - **Open Source**: Licensed under Apache 2.0, encouraging unrestricted community contributions and commercial usage.
 - **User-Friendly**: Designed with simplicity in mind for easy deployment and management.
@@ -54,6 +55,7 @@ Unlike other storage systems, RustFS is released under the permissible Apache 2.
 | **Logging**             | ✅ Available | **Lifecycle Management** | 🚧 Under Testing |
 | **Event Notifications** | ✅ Available | **Distributed Mode**     | 🚧 Under Testing |
 | **K8s Helm Charts**     | ✅ Available | **RustFS KMS**           | 🚧 Under Testing |
+| **Keystone Auth**       | ✅ Available | **Multi-Tenancy**        | ✅ Available     |

 ## RustFS vs MinIO Performance

@@ -37,8 +37,11 @@ datas = "datas"
 bre = "bre"
 abd = "abd"
 mak = "mak"
+gae = "gae"
+GAE = "GAE"
 # s3-tests original test names (cannot be changed)
 nonexisted = "nonexisted"
+consts = "consts"

 [files]
 extend-exclude = []
@@ -19,8 +19,9 @@ use rumqttc::QoS;
 use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
 use rustfs_config::{
    AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT,
-    WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
+    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
+    WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
+    WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
@@ -75,6 +76,11 @@ impl TargetFactory for WebhookTargetFactory {
                .unwrap_or(DEFAULT_LIMIT),
            client_cert: config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(),
            client_key: config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(),
+            client_ca: config.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
+            skip_tls_verify: config
+                .lookup(WEBHOOK_SKIP_TLS_VERIFY)
+                .and_then(|v| v.parse::<bool>().ok())
+                .unwrap_or(RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT),
            target_type: rustfs_targets::target::TargetType::AuditLog,
        };

@@ -32,6 +32,7 @@ tokio = { workspace = true }
 tonic = { workspace = true }
 uuid = { workspace = true }
 chrono = { workspace = true }
+metrics = { workspace = true }
 rustfs-madmin = { workspace = true }
 rustfs-filemeta = { workspace = true }
 serde = { workspace = true }
@@ -39,4 +40,4 @@ path-clean = { workspace = true }
 rmp-serde = { workspace = true }
 async-trait = { workspace = true }
 s3s = { workspace = true }
-tracing = { workspace = true }
+tracing = { workspace = true }
@@ -38,6 +38,14 @@ pub async fn set_global_local_node_name(name: &str) {
    *GLOBAL_LOCAL_NODE_NAME.write().await = name.to_string();
 }

+/// Get the global local node name.
+///
+/// # Returns
+/// * `String` - The local node name.
+pub async fn get_global_local_node_name() -> String {
+    GLOBAL_LOCAL_NODE_NAME.read().await.clone()
+}
+
 /// Set the global RustFS initialization time to the current UTC time.
 pub async fn set_global_init_time_now() {
    let now = Utc::now();
@@ -14,7 +14,8 @@

 use crate::last_minute::{AccElem, LastMinuteLatency};
 use chrono::{DateTime, Utc};
-use rustfs_madmin::metrics::ScannerMetrics as M_ScannerMetrics;
+use rustfs_madmin::metrics::{ScannerMetrics as M_ScannerMetrics, TimedAction};
+use serde::{Deserialize, Serialize};
 use std::{
    collections::HashMap,
    fmt::Display,
@@ -64,6 +65,37 @@ impl IlmAction {
            || *self == Self::DeleteAllVersionsAction
            || *self == Self::DelMarkerDeleteAllVersionsAction
    }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::NoneAction => "none",
+            Self::DeleteAction => "delete",
+            Self::DeleteVersionAction => "delete_version",
+            Self::TransitionAction => "transition",
+            Self::TransitionVersionAction => "transition_version",
+            Self::DeleteRestoredAction => "delete_restored",
+            Self::DeleteRestoredVersionAction => "delete_restored_version",
+            Self::DeleteAllVersionsAction => "delete_all_versions",
+            Self::DelMarkerDeleteAllVersionsAction => "del_marker_delete_all_versions",
+            Self::ActionCount => "action_count",
+        }
+    }
+
+    pub fn from_index(i: usize) -> Option<Self> {
+        match i {
+            0 => Some(Self::NoneAction),
+            1 => Some(Self::DeleteAction),
+            2 => Some(Self::DeleteVersionAction),
+            3 => Some(Self::TransitionAction),
+            4 => Some(Self::TransitionVersionAction),
+            5 => Some(Self::DeleteRestoredAction),
+            6 => Some(Self::DeleteRestoredVersionAction),
+            7 => Some(Self::DeleteAllVersionsAction),
+            8 => Some(Self::DelMarkerDeleteAllVersionsAction),
+            9 => Some(Self::ActionCount),
+            _ => None,
+        }
+    }
 }

 impl Display for IlmAction {
@@ -272,14 +304,76 @@ pub struct Metrics {
    cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
 }

-// This is a placeholder. We'll need to define this struct.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct CurrentCycle {
    pub current: u64,
+    pub next: u64,
    pub cycle_completed: Vec<DateTime<Utc>>,
    pub started: DateTime<Utc>,
 }

+impl CurrentCycle {
+    pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        *self = rmp_serde::from_slice(buf)?;
+        Ok(())
+    }
+
+    pub fn marshal(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
+        Ok(rmp_serde::to_vec(self)?)
+    }
+}
+
+/// OTEL metric name constants for scanner metrics
+const OTEL_SCANNER_OBJECTS_SCANNED: &str = "rustfs_scanner_objects_scanned_total";
+const OTEL_SCANNER_DIRECTORIES_SCANNED: &str = "rustfs_scanner_directories_scanned_total";
+const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
+const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
+const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
+const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
+
+/// Emit an OTEL counter increment for the given scanner metric.
+/// ScanCycle and ScanBucketDrive are handled by dedicated emit functions with labels.
+fn emit_otel_counter(metric: usize, count: u64) {
+    match Metric::from_index(metric) {
+        Some(Metric::ScanObject) => {
+            metrics::counter!(OTEL_SCANNER_OBJECTS_SCANNED).increment(count);
+        }
+        Some(Metric::ScanFolder) => {
+            metrics::counter!(OTEL_SCANNER_DIRECTORIES_SCANNED).increment(count);
+        }
+        _ => {}
+    }
+}
+
+/// Emit OTel metrics for a completed scan cycle.
+/// Counter with result label + gauge for last successful cycle duration.
+pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
+    let result = if success { "success" } else { "error" };
+    metrics::counter!(OTEL_SCANNER_CYCLES, "result" => result).increment(1);
+    if success {
+        metrics::gauge!(OTEL_SCANNER_CYCLE_DURATION_SECONDS).set(duration.as_secs_f64());
+    }
+}
+
+/// Emit OTel metrics for a completed bucket-drive scan.
+/// Counter with result/bucket/disk labels + histogram for duration.
+pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
+    let result = if success { "success" } else { "error" };
+    metrics::counter!(
+        OTEL_SCANNER_BUCKETS_SCANNED,
+        "result" => result,
+        "bucket" => bucket.to_owned(),
+        "disk" => disk.to_owned()
+    )
+    .increment(1);
+    metrics::histogram!(
+        OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS,
+        "bucket" => bucket.to_owned(),
+        "disk" => disk.to_owned()
+    )
+    .record(duration.as_secs_f64());
+}
+
 impl Metrics {
    pub fn new() -> Self {
        let operations = (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect();
@@ -307,6 +401,7 @@ impl Metrics {

            // Update operation count
            global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric, 1);

            // Update latency for realtime metrics (spawn async task for this)
            if (metric) < Metric::LastRealtime as usize {
@@ -332,6 +427,7 @@ impl Metrics {

            // Update operation count
            global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric, 1);

            // Update latency for realtime metrics with size (spawn async task)
            if (metric) < Metric::LastRealtime as usize {
@@ -352,6 +448,7 @@ impl Metrics {

            // Update operation count
            global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric, 1);

            // Update latency for realtime metrics (spawn async task)
            if (metric) < Metric::LastRealtime as usize {
@@ -373,6 +470,7 @@ impl Metrics {

                // Update operation count
                global_metrics().operations[metric].fetch_add(count as u64, Ordering::Relaxed);
+                emit_otel_counter(metric, count as u64);

                // Update latency for realtime metrics (spawn async task)
                if (metric) < Metric::LastRealtime as usize {
@@ -408,6 +506,7 @@ impl Metrics {
        let metric = metric as usize;
        // Update operation count
        global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+        emit_otel_counter(metric, 1);

        // Update latency for realtime metrics
        if (metric) < Metric::LastRealtime as usize {
@@ -494,10 +593,43 @@ impl Metrics {
        for i in 0..Metric::LastRealtime as usize {
            let last_min = self.latency[i].total().await;
            if last_min.n > 0
-                && let Some(_metric) = Metric::from_index(i)
+                && let Some(metric) = Metric::from_index(i)
            {
-                // Convert to madmin TimedAction format if needed
-                // This would require implementing the conversion
+                metrics.last_minute.actions.insert(
+                    metric.as_str().to_string(),
+                    TimedAction {
+                        count: last_min.n,
+                        acc_time: last_min.total,
+                        bytes: last_min.size,
+                    },
+                );
+            }
+        }
+
+        // Lifetime ILM operations
+        for i in 0..IlmAction::ActionCount as usize {
+            let count = self.actions[i].load(Ordering::Relaxed);
+            if count > 0
+                && let Some(action) = IlmAction::from_index(i)
+            {
+                metrics.life_time_ilm.insert(action.as_str().to_string(), count);
+            }
+        }
+
+        // Last minute ILM latency
+        for i in 0..IlmAction::ActionCount as usize {
+            let last_min = self.actions_latency[i].total().await;
+            if last_min.n > 0
+                && let Some(action) = IlmAction::from_index(i)
+            {
+                metrics.last_minute.ilm.insert(
+                    action.as_str().to_string(),
+                    TimedAction {
+                        count: last_min.n,
+                        acc_time: last_min.total,
+                        bytes: last_min.size,
+                    },
+                );
            }
        }

@@ -550,3 +682,31 @@ impl Default for Metrics {
        Self::new()
    }
 }
+
+pub struct CloseDiskGuard(CloseDiskFn);
+
+impl CloseDiskGuard {
+    pub fn new(close_disk: CloseDiskFn) -> Self {
+        Self(close_disk)
+    }
+
+    pub async fn close(&self) {
+        self.0().await;
+    }
+}
+
+impl Drop for CloseDiskGuard {
+    fn drop(&mut self) {
+        // Drop cannot be async, so we spawn the async cleanup task
+        // The task will run in the background and complete asynchronously
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            let close_fn = self.0.clone();
+            handle.spawn(async move {
+                close_fn().await;
+            });
+        } else {
+            // If we're not in a tokio runtime context, we can't spawn
+            // This is a best-effort cleanup, so we just skip it
+        }
+    }
+}
@@ -32,6 +32,24 @@

 For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).

+## Environment Variable Naming Conventions
+
+RustFS uses a flat naming style for top-level configuration: environment variables are `RUSTFS_*` without nested module segments.
+
+Examples:
+- `RUSTFS_REGION`
+- `RUSTFS_ADDRESS`
+- `RUSTFS_VOLUMES`
+- `RUSTFS_LICENSE`
+
+Current guidance:
+- Prefer module-specific names only when they are not top-level product configuration.
+- Renamed variables must keep backward-compatible aliases until before beta.
+- Alias usage must emit deprecation warnings and be treated as transitional only.
+- Deprecated example:
+  - `RUSTFS_ENABLE_SCANNER` -> `RUSTFS_SCANNER_ENABLED`
+  - `RUSTFS_ENABLE_HEAL` -> `RUSTFS_HEAL_ENABLED`
+
 ## 📄 License

 This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
@@ -20,9 +20,11 @@ pub const ENV_AUDIT_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_WEBHOOK_QUEUE_LIMI
 pub const ENV_AUDIT_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR";
 pub const ENV_AUDIT_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_CERT";
 pub const ENV_AUDIT_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_KEY";
+pub const ENV_AUDIT_WEBHOOK_CLIENT_CA: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_CA";
+pub const ENV_AUDIT_WEBHOOK_SKIP_TLS_VERIFY: &str = "RUSTFS_AUDIT_WEBHOOK_SKIP_TLS_VERIFY";

 /// List of all environment variable keys for a webhook target.
-pub const ENV_AUDIT_WEBHOOK_KEYS: &[&str; 7] = &[
+pub const ENV_AUDIT_WEBHOOK_KEYS: &[&str; 9] = &[
    ENV_AUDIT_WEBHOOK_ENABLE,
    ENV_AUDIT_WEBHOOK_ENDPOINT,
    ENV_AUDIT_WEBHOOK_AUTH_TOKEN,
@@ -30,6 +32,8 @@ pub const ENV_AUDIT_WEBHOOK_KEYS: &[&str; 7] = &[
    ENV_AUDIT_WEBHOOK_QUEUE_DIR,
    ENV_AUDIT_WEBHOOK_CLIENT_CERT,
    ENV_AUDIT_WEBHOOK_CLIENT_KEY,
+    ENV_AUDIT_WEBHOOK_CLIENT_CA,
+    ENV_AUDIT_WEBHOOK_SKIP_TLS_VERIFY,
 ];

 /// A list of all valid configuration keys for a webhook target.
@@ -42,4 +46,6 @@ pub const AUDIT_WEBHOOK_KEYS: &[&str] = &[
    crate::WEBHOOK_CLIENT_CERT,
    crate::WEBHOOK_CLIENT_KEY,
    crate::COMMENT_KEY,
+    crate::WEBHOOK_CLIENT_CA,
+    crate::WEBHOOK_SKIP_TLS_VERIFY,
 ];
@@ -62,8 +62,8 @@ pub const DEFAULT_CONSOLE_ENABLE: bool = true;
 /// Default OBS configuration endpoint
 /// Environment variable: DEFAULT_OBS_ENDPOINT
 /// Command line argument: --obs-endpoint
-/// Example: DEFAULT_OBS_ENDPOINT="http://localost:4317"
-/// Example: --obs-endpoint http://localost:4317
+/// Example: DEFAULT_OBS_ENDPOINT="http://localost:4318"
+/// Example: --obs-endpoint http://localost:4318
 pub const DEFAULT_OBS_ENDPOINT: &str = "";

 /// Default TLS key for rustfs
@@ -98,6 +98,39 @@ pub const RUSTFS_HTTP_PREFIX: &str = "http://";
 /// Default value: https://
 pub const RUSTFS_HTTPS_PREFIX: &str = "https://";

+/// Default documentation URL for rustfs
+/// This is the default documentation URL for rustfs.
+/// It is used to provide the documentation of the application.
+/// Default value: https://docs.rustfs.com
+pub const RUSTFS_DOCS_URL: &str = "https://docs.rustfs.com";
+
+/// Default GitHub URL for rustfs
+/// This is the default GitHub URL for rustfs.
+/// It is used to provide the source code of the application.
+/// Default value: https://github.com/rustfs/rustfs
+pub const RUSTFS_GITHUB_URL: &str = "https://github.com/rustfs/rustfs";
+
+/// Default license for rustfs
+/// This is the default license for rustfs.
+/// It is used to provide the license of the application.
+/// Default value: Apache-2.0
+pub const RUSTFS_LICENSE: &str = "Apache-2.0";
+
+/// Default license URL for rustfs
+/// This is the default license URL for rustfs.
+/// It is used to provide the license URL of the application.
+/// Default value: https://www.apache.org/licenses/LICENSE-2.0
+pub const RUSTFS_LICENSE_URL: &str = "https://www.apache.org/licenses/LICENSE-2.0";
+
+/// Environment variable for rustfs address
+/// This is the environment variable for rustfs address.
+/// It is used to bind the server to a specific address.
+/// Example: RUSTFS_ADDRESS=":9000"
+pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
+
+/// Environment variable for server volumes.
+pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
+
 /// Default port for rustfs
 /// This is the default port for rustfs.
 /// This is used to bind the server to a specific port.
@@ -113,20 +146,34 @@ pub const DEFAULT_CONSOLE_PORT: u16 = 9001;

 /// Default address for rustfs console
 /// This is the default address for rustfs console.
+/// This is used to bind the console server to a specific address.
+/// Default value: :9001
 pub const DEFAULT_CONSOLE_ADDRESS: &str = concat!(":", DEFAULT_CONSOLE_PORT);

+/// Default region for rustfs
+/// This is the default region for rustfs.
+/// It is used to identify the region of the application.
+/// Default value: us-east-1
+pub const RUSTFS_REGION: &str = "us-east-1";
+
+/// Environment variable for server region.
+pub const ENV_RUSTFS_REGION: &str = "RUSTFS_REGION";
+
+/// Environment variable for server license.
+pub const ENV_RUSTFS_LICENSE: &str = "RUSTFS_LICENSE";
+
 /// Default log filename for rustfs
 /// This is the default log filename for rustfs.
 /// It is used to store the logs of the application.
 /// Default value: rustfs.log
-/// Environment variable: RUSTFS_OBSERVABILITY_LOG_FILENAME
+/// Environment variable: RUSTFS_OBS_LOG_FILENAME
 pub const DEFAULT_LOG_FILENAME: &str = "rustfs";

 /// Default OBS log filename for rustfs
 /// This is the default log filename for OBS.
 /// It is used to store the logs of the application.
 /// Default value: rustfs.log
-pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, "");
+pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, ".log");

 /// Default log directory for rustfs
 /// This is the default log directory for rustfs.
@@ -145,9 +192,9 @@ pub const DEFAULT_LOG_ROTATION_SIZE_MB: u64 = 100;
 /// Default log rotation time for rustfs
 /// This is the default log rotation time for rustfs.
 /// It is used to rotate the logs of the application.
-/// Default value: hour, eg: day,hour,minute,second
+/// Default value: hour, eg: daily,hourly,minutely
 /// Environment variable: RUSTFS_OBS_LOG_ROTATION_TIME
-pub const DEFAULT_LOG_ROTATION_TIME: &str = "hour";
+pub const DEFAULT_LOG_ROTATION_TIME: &str = "hourly";

 /// Default log keep files for rustfs
 /// This is the default log keep files for rustfs.
@@ -156,11 +203,29 @@ pub const DEFAULT_LOG_ROTATION_TIME: &str = "hour";
 /// Environment variable: RUSTFS_OBS_LOG_KEEP_FILES
 pub const DEFAULT_LOG_KEEP_FILES: usize = 30;

+/// Default trace export enabled
+/// It is used to enable or disable exporting traces
+/// Default value: true
+/// Environment variable: RUSTFS_OBS_TRACES_EXPORT_ENABLED
+pub const DEFAULT_OBS_TRACES_EXPORT_ENABLED: bool = true;
+
+/// Default metrics export enabled
+/// It is used to enable or disable exporting metrics
+/// Default value: true
+/// Environment variable: RUSTFS_OBS_METRICS_EXPORT_ENABLED
+pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true;
+
+/// Default logs export enabled
+/// It is used to enable or disable exporting logs
+/// Default value: true
+/// Environment variable: RUSTFS_OBS_LOGS_EXPORT_ENABLED
+pub const DEFAULT_OBS_LOGS_EXPORT_ENABLED: bool = true;
+
 /// Default log local logging enabled for rustfs
 /// This is the default log local logging enabled for rustfs.
 /// It is used to enable or disable local logging of the application.
 /// Default value: false
-/// Environment variable: RUSTFS_OBS_LOGL_STDOUT_ENABLED
+/// Environment variable: RUSTFS_OBS_LOG_STDOUT_ENABLED
 pub const DEFAULT_OBS_LOG_STDOUT_ENABLED: bool = false;

 /// Constant representing 1 Kibibyte (1024 bytes)
@@ -20,6 +20,8 @@ pub const EVENT_DEFAULT_DIR: &str = "/opt/rustfs/events"; // Default directory f
 pub const AUDIT_DEFAULT_DIR: &str = "/opt/rustfs/audit"; // Default directory for audit store
 pub const DEFAULT_LIMIT: u64 = 100000; // Default store limit

+pub const RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT: bool = false;
+
 /// Standard config keys and values.
 pub const ENABLE_KEY: &str = "enable";
 pub const COMMENT_KEY: &str = "comment";
@@ -19,8 +19,10 @@ pub(crate) mod console;
 pub(crate) mod env;
 pub(crate) mod heal;
 pub(crate) mod object;
+pub(crate) mod oidc;
 pub(crate) mod profiler;
 pub(crate) mod protocols;
+pub(crate) mod proxy;
 pub(crate) mod quota;
 pub(crate) mod runtime;
 pub(crate) mod scanner;
@@ -0,0 +1,91 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// OIDC configuration field keys (used in KVS)
+pub const OIDC_CONFIG_URL: &str = "config_url";
+pub const OIDC_CLIENT_ID: &str = "client_id";
+pub const OIDC_CLIENT_SECRET: &str = "client_secret";
+pub const OIDC_SCOPES: &str = "scopes";
+pub const OIDC_REDIRECT_URI: &str = "redirect_uri";
+pub const OIDC_REDIRECT_URI_DYNAMIC: &str = "redirect_uri_dynamic";
+pub const OIDC_CLAIM_NAME: &str = "claim_name";
+pub const OIDC_CLAIM_PREFIX: &str = "claim_prefix";
+pub const OIDC_ROLE_POLICY: &str = "role_policy";
+pub const OIDC_DISPLAY_NAME: &str = "display_name";
+pub const OIDC_GROUPS_CLAIM: &str = "groups_claim";
+pub const OIDC_EMAIL_CLAIM: &str = "email_claim";
+pub const OIDC_USERNAME_CLAIM: &str = "username_claim";
+
+// Environment variable names for OIDC
+pub const ENV_IDENTITY_OPENID_ENABLE: &str = "RUSTFS_IDENTITY_OPENID_ENABLE";
+pub const ENV_IDENTITY_OPENID_CONFIG_URL: &str = "RUSTFS_IDENTITY_OPENID_CONFIG_URL";
+pub const ENV_IDENTITY_OPENID_CLIENT_ID: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_ID";
+pub const ENV_IDENTITY_OPENID_CLIENT_SECRET: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_SECRET";
+pub const ENV_IDENTITY_OPENID_SCOPES: &str = "RUSTFS_IDENTITY_OPENID_SCOPES";
+pub const ENV_IDENTITY_OPENID_REDIRECT_URI: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI";
+pub const ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC";
+pub const ENV_IDENTITY_OPENID_CLAIM_NAME: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_NAME";
+pub const ENV_IDENTITY_OPENID_CLAIM_PREFIX: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_PREFIX";
+pub const ENV_IDENTITY_OPENID_ROLE_POLICY: &str = "RUSTFS_IDENTITY_OPENID_ROLE_POLICY";
+pub const ENV_IDENTITY_OPENID_DISPLAY_NAME: &str = "RUSTFS_IDENTITY_OPENID_DISPLAY_NAME";
+pub const ENV_IDENTITY_OPENID_GROUPS_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM";
+pub const ENV_IDENTITY_OPENID_EMAIL_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM";
+pub const ENV_IDENTITY_OPENID_USERNAME_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM";
+
+/// List of all environment variable keys for an OIDC provider.
+pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
+    ENV_IDENTITY_OPENID_ENABLE,
+    ENV_IDENTITY_OPENID_CONFIG_URL,
+    ENV_IDENTITY_OPENID_CLIENT_ID,
+    ENV_IDENTITY_OPENID_CLIENT_SECRET,
+    ENV_IDENTITY_OPENID_SCOPES,
+    ENV_IDENTITY_OPENID_REDIRECT_URI,
+    ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC,
+    ENV_IDENTITY_OPENID_CLAIM_NAME,
+    ENV_IDENTITY_OPENID_CLAIM_PREFIX,
+    ENV_IDENTITY_OPENID_ROLE_POLICY,
+    ENV_IDENTITY_OPENID_DISPLAY_NAME,
+    ENV_IDENTITY_OPENID_GROUPS_CLAIM,
+    ENV_IDENTITY_OPENID_EMAIL_CLAIM,
+    ENV_IDENTITY_OPENID_USERNAME_CLAIM,
+];
+
+/// A list of all valid configuration keys for an OIDC provider.
+pub const IDENTITY_OPENID_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    OIDC_CONFIG_URL,
+    OIDC_CLIENT_ID,
+    OIDC_CLIENT_SECRET,
+    OIDC_SCOPES,
+    OIDC_REDIRECT_URI,
+    OIDC_REDIRECT_URI_DYNAMIC,
+    OIDC_CLAIM_NAME,
+    OIDC_CLAIM_PREFIX,
+    OIDC_ROLE_POLICY,
+    OIDC_DISPLAY_NAME,
+    OIDC_GROUPS_CLAIM,
+    OIDC_EMAIL_CLAIM,
+    OIDC_USERNAME_CLAIM,
+    crate::COMMENT_KEY,
+];
+
+// Default values
+pub const OIDC_DEFAULT_SCOPES: &str = "openid,profile,email";
+pub const OIDC_DEFAULT_CLAIM_NAME: &str = "groups";
+pub const OIDC_DEFAULT_GROUPS_CLAIM: &str = "groups";
+pub const OIDC_DEFAULT_EMAIL_CLAIM: &str = "email";
+pub const OIDC_DEFAULT_USERNAME_CLAIM: &str = "preferred_username";
+
+// Subsystem identifier
+pub const IDENTITY_OPENID_SUB_SYS: &str = "identity_openid";
@@ -14,27 +14,34 @@

 //! Protocol server configuration constants

-/// Default FTPS server bind address
-pub const DEFAULT_FTPS_ADDRESS: &str = "0.0.0.0:8021";
+/// Default FTP server bind address (non-encrypted)
+pub const DEFAULT_FTP_ADDRESS: &str = "0.0.0.0:8021";

-/// Default SFTP server bind address
-pub const DEFAULT_SFTP_ADDRESS: &str = "0.0.0.0:8022";
+/// Default FTPS server bind address (FTP over TLS)
+pub const DEFAULT_FTPS_ADDRESS: &str = "0.0.0.0:8022";
+
+/// Default FTP passive ports range (optional)
+pub const DEFAULT_FTP_PASSIVE_PORTS: Option<&str> = None;

 /// Default FTPS passive ports range (optional)
 pub const DEFAULT_FTPS_PASSIVE_PORTS: Option<&str> = None;

+/// Default FTP external IP (auto-detected)
+pub const DEFAULT_FTP_EXTERNAL_IP: Option<&str> = None;
+
 /// Default FTPS external IP (auto-detected)
 pub const DEFAULT_FTPS_EXTERNAL_IP: Option<&str> = None;

 /// Environment variable names
+pub const ENV_FTP_ENABLE: &str = "RUSTFS_FTP_ENABLE";
+pub const ENV_FTP_ADDRESS: &str = "RUSTFS_FTP_ADDRESS";
+pub const ENV_FTP_PASSIVE_PORTS: &str = "RUSTFS_FTP_PASSIVE_PORTS";
+pub const ENV_FTP_EXTERNAL_IP: &str = "RUSTFS_FTP_EXTERNAL_IP";
+
 pub const ENV_FTPS_ENABLE: &str = "RUSTFS_FTPS_ENABLE";
 pub const ENV_FTPS_ADDRESS: &str = "RUSTFS_FTPS_ADDRESS";
-pub const ENV_FTPS_CERTS_FILE: &str = "RUSTFS_FTPS_CERTS_FILE";
-pub const ENV_FTPS_KEY_FILE: &str = "RUSTFS_FTPS_KEY_FILE";
+pub const ENV_FTPS_TLS_ENABLED: &str = "RUSTFS_FTPS_TLS_ENABLED";
+pub const ENV_FTPS_CERTS_DIR: &str = "RUSTFS_FTPS_CERTS_DIR";
+pub const ENV_FTPS_CA_FILE: &str = "RUSTFS_FTPS_CA_FILE";
 pub const ENV_FTPS_PASSIVE_PORTS: &str = "RUSTFS_FTPS_PASSIVE_PORTS";
 pub const ENV_FTPS_EXTERNAL_IP: &str = "RUSTFS_FTPS_EXTERNAL_IP";
-
-pub const ENV_SFTP_ENABLE: &str = "RUSTFS_SFTP_ENABLE";
-pub const ENV_SFTP_ADDRESS: &str = "RUSTFS_SFTP_ADDRESS";
-pub const ENV_SFTP_HOST_KEY: &str = "RUSTFS_SFTP_HOST_KEY";
-pub const ENV_SFTP_AUTHORIZED_KEYS: &str = "RUSTFS_SFTP_AUTHORIZED_KEYS";
@@ -0,0 +1,125 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::DEFAULT_LOG_LEVEL;
+
+// ==================== Base Proxy Configuration ====================
+/// Environment variable to enable the trusted proxy middleware.
+pub const ENV_TRUSTED_PROXY_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_ENABLED";
+/// Trusted proxy middleware is enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_ENABLED: bool = true;
+
+/// Environment variable for the proxy validation mode.
+pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDATION_MODE";
+/// Default validation mode is "hop_by_hop".
+pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
+
+/// Environment variable to enable RFC 7239 "Forwarded" header support.
+pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
+/// RFC 7239 support is enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
+
+/// Environment variable for the maximum allowed proxy hops.
+pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
+/// Default maximum hops is 10.
+pub const DEFAULT_TRUSTED_PROXY_MAX_HOPS: usize = 10;
+
+/// Environment variable to enable proxy chain continuity checks.
+pub const ENV_TRUSTED_PROXY_CHAIN_CONTINUITY_CHECK: &str = "RUSTFS_TRUSTED_PROXY_CHAIN_CONTINUITY_CHECK";
+/// Continuity checks are enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_CHAIN_CONTINUITY_CHECK: bool = true;
+
+/// Environment variable to enable logging of failed proxy validations.
+pub const ENV_TRUSTED_PROXY_LOG_FAILED_VALIDATIONS: &str = "RUSTFS_TRUSTED_PROXY_LOG_FAILED_VALIDATIONS";
+/// Logging of failed validations is enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_LOG_FAILED_VALIDATIONS: bool = true;
+
+// ==================== Trusted Proxy Networks ====================
+/// Environment variable for the list of trusted proxy networks (comma-separated IP/CIDR).
+pub const ENV_TRUSTED_PROXY_PROXIES: &str = "RUSTFS_TRUSTED_PROXY_NETWORKS";
+/// Default trusted networks include localhost and common private ranges.
+pub const DEFAULT_TRUSTED_PROXY_PROXIES: &str = "127.0.0.1,::1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fd00::/8";
+
+/// Environment variable for additional trusted proxy networks (production specific).
+pub const ENV_TRUSTED_PROXY_EXTRA_PROXIES: &str = "RUSTFS_TRUSTED_PROXY_EXTRA_NETWORKS";
+/// No extra trusted networks by default.
+pub const DEFAULT_TRUSTED_PROXY_EXTRA_PROXIES: &str = "";
+
+/// Environment variable for individual trusted proxy IPs.
+pub const ENV_TRUSTED_PROXY_IPS: &str = "RUSTFS_TRUSTED_PROXY_IPS";
+/// No individual trusted IPs by default.
+pub const DEFAULT_TRUSTED_PROXY_IPS: &str = "";
+
+/// Environment variable for private network ranges used in internal validation.
+pub const ENV_TRUSTED_PROXY_PRIVATE_NETWORKS: &str = "RUSTFS_TRUSTED_PROXY_PRIVATE_NETWORKS";
+/// Default private networks include common RFC 1918 and RFC 4193 ranges.
+pub const DEFAULT_TRUSTED_PROXY_PRIVATE_NETWORKS: &str = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fd00::/8";
+
+// ==================== Cache Configuration ====================
+/// Environment variable for the proxy validation cache capacity.
+pub const ENV_TRUSTED_PROXY_CACHE_CAPACITY: &str = "RUSTFS_TRUSTED_PROXY_CACHE_CAPACITY";
+/// Default cache capacity is 10,000 entries.
+pub const DEFAULT_TRUSTED_PROXY_CACHE_CAPACITY: usize = 10_000;
+
+/// Environment variable for the cache entry time-to-live (TTL) in seconds.
+pub const ENV_TRUSTED_PROXY_CACHE_TTL_SECONDS: &str = "RUSTFS_TRUSTED_PROXY_CACHE_TTL_SECONDS";
+/// Default cache TTL is 300 seconds (5 minutes).
+pub const DEFAULT_TRUSTED_PROXY_CACHE_TTL_SECONDS: u64 = 300;
+
+/// Environment variable for the cache cleanup interval in seconds.
+pub const ENV_TRUSTED_PROXY_CACHE_CLEANUP_INTERVAL: &str = "RUSTFS_TRUSTED_PROXY_CACHE_CLEANUP_INTERVAL";
+/// Default cleanup interval is 60 seconds.
+pub const DEFAULT_TRUSTED_PROXY_CACHE_CLEANUP_INTERVAL: u64 = 60;
+
+// ==================== Monitoring Configuration ====================
+/// Environment variable to enable Prometheus metrics.
+pub const ENV_TRUSTED_PROXY_METRICS_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_METRICS_ENABLED";
+/// Metrics are enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_METRICS_ENABLED: bool = true;
+
+/// Environment variable for the application log level.
+pub const ENV_TRUSTED_PROXIES_LOG_LEVEL: &str = "RUSTFS_TRUSTED_PROXY_LOG_LEVEL";
+/// Default log level is "info".
+pub const DEFAULT_TRUSTED_PROXIES_LOG_LEVEL: &str = DEFAULT_LOG_LEVEL;
+
+/// Environment variable to enable structured JSON logging.
+pub const ENV_TRUSTED_PROXY_STRUCTURED_LOGGING: &str = "RUSTFS_TRUSTED_PROXY_STRUCTURED_LOGGING";
+/// Structured logging is disabled by default.
+pub const DEFAULT_TRUSTED_PROXY_STRUCTURED_LOGGING: bool = false;
+
+/// Environment variable to enable distributed tracing.
+pub const ENV_TRUSTED_PROXY_TRACING_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_TRACING_ENABLED";
+/// Tracing is enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_TRACING_ENABLED: bool = true;
+
+// ==================== Cloud Integration ====================
+/// Environment variable to enable automatic cloud metadata discovery.
+pub const ENV_TRUSTED_PROXY_CLOUD_METADATA_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_CLOUD_METADATA_ENABLED";
+/// Cloud metadata discovery is disabled by default.
+pub const DEFAULT_TRUSTED_PROXY_CLOUD_METADATA_ENABLED: bool = false;
+
+/// Environment variable for the cloud metadata request timeout in seconds.
+pub const ENV_TRUSTED_PROXY_CLOUD_METADATA_TIMEOUT: &str = "RUSTFS_TRUSTED_PROXY_CLOUD_METADATA_TIMEOUT";
+/// Default cloud metadata timeout is 5 seconds.
+pub const DEFAULT_TRUSTED_PROXY_CLOUD_METADATA_TIMEOUT: u64 = 5;
+
+/// Environment variable to enable Cloudflare IP range integration.
+pub const ENV_TRUSTED_PROXY_CLOUDFLARE_IPS_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_CLOUDFLARE_IPS_ENABLED";
+/// Cloudflare integration is disabled by default.
+pub const DEFAULT_TRUSTED_PROXY_CLOUDFLARE_IPS_ENABLED: bool = false;
+
+/// Environment variable to force a specific cloud provider (overrides auto-detection).
+pub const ENV_TRUSTED_PROXY_CLOUD_PROVIDER_FORCE: &str = "RUSTFS_TRUSTED_PROXY_CLOUD_PROVIDER_FORCE";
+/// No forced provider by default.
+pub const DEFAULT_TRUSTED_PROXY_CLOUD_PROVIDER_FORCE: &str = "";
@@ -16,6 +16,8 @@ pub const WEBHOOK_ENDPOINT: &str = "endpoint";
 pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token";
 pub const WEBHOOK_CLIENT_CERT: &str = "client_cert";
 pub const WEBHOOK_CLIENT_KEY: &str = "client_key";
+pub const WEBHOOK_CLIENT_CA: &str = "client_ca";
+pub const WEBHOOK_SKIP_TLS_VERIFY: &str = "skip_tls_verify";
 pub const WEBHOOK_BATCH_SIZE: &str = "batch_size";
 pub const WEBHOOK_QUEUE_LIMIT: &str = "queue_limit";
 pub const WEBHOOK_QUEUE_DIR: &str = "queue_dir";
@@ -33,6 +33,8 @@ pub use constants::profiler::*;
 #[cfg(feature = "constants")]
 pub use constants::protocols::*;
 #[cfg(feature = "constants")]
+pub use constants::proxy::*;
+#[cfg(feature = "constants")]
 pub use constants::quota::*;
 #[cfg(feature = "constants")]
 pub use constants::runtime::*;
@@ -42,6 +44,10 @@ pub use constants::scanner::*;
 pub use constants::targets::*;
 #[cfg(feature = "constants")]
 pub use constants::tls::*;
+#[cfg(feature = "constants")]
+pub mod oidc {
+    pub use super::constants::oidc::*;
+}
 #[cfg(feature = "audit")]
 pub mod audit;
 #[cfg(feature = "notify")]
@@ -22,6 +22,8 @@ pub const NOTIFY_WEBHOOK_KEYS: &[&str] = &[
    crate::WEBHOOK_CLIENT_CERT,
    crate::WEBHOOK_CLIENT_KEY,
    crate::COMMENT_KEY,
+    crate::WEBHOOK_CLIENT_CA,
+    crate::WEBHOOK_SKIP_TLS_VERIFY,
 ];

 // Webhook Environment Variables
@@ -32,8 +34,10 @@ pub const ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_LI
 pub const ENV_NOTIFY_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR";
 pub const ENV_NOTIFY_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CERT";
 pub const ENV_NOTIFY_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_KEY";
+pub const ENV_NOTIFY_WEBHOOK_CLIENT_CA: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CA";
+pub const ENV_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY: &str = "RUSTFS_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY";

-pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 7] = &[
+pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 9] = &[
    ENV_NOTIFY_WEBHOOK_ENABLE,
    ENV_NOTIFY_WEBHOOK_ENDPOINT,
    ENV_NOTIFY_WEBHOOK_AUTH_TOKEN,
@@ -41,4 +45,6 @@ pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 7] = &[
    ENV_NOTIFY_WEBHOOK_QUEUE_DIR,
    ENV_NOTIFY_WEBHOOK_CLIENT_CERT,
    ENV_NOTIFY_WEBHOOK_CLIENT_KEY,
+    ENV_NOTIFY_WEBHOOK_CLIENT_CA,
+    ENV_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY,
 ];
@@ -15,6 +15,7 @@
 // Observability Keys

 mod metrics;
+use const_str::concat;
 pub use metrics::*;

 pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
@@ -27,6 +28,12 @@ pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
 pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
 pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
 pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
+
+/// Per-signal enable/disable flags (default: true)
+pub const ENV_OBS_TRACES_EXPORT_ENABLED: &str = "RUSTFS_OBS_TRACES_EXPORT_ENABLED";
+pub const ENV_OBS_METRICS_EXPORT_ENABLED: &str = "RUSTFS_OBS_METRICS_EXPORT_ENABLED";
+pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED";
+
 pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
 pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
 pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
@@ -35,23 +42,32 @@ pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB"
 pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
 pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";

-/// Log pool capacity for async logging
-pub const ENV_OBS_LOG_POOL_CAPA: &str = "RUSTFS_OBS_LOG_POOL_CAPA";
+/// Log cleanup related configurations
+pub const ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES";
+pub const ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES";
+pub const ENV_OBS_LOG_COMPRESS_OLD_FILES: &str = "RUSTFS_OBS_LOG_COMPRESS_OLD_FILES";
+pub const ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL: &str = "RUSTFS_OBS_LOG_GZIP_COMPRESSION_LEVEL";
+pub const ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS: &str = "RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS";
+pub const ENV_OBS_LOG_EXCLUDE_PATTERNS: &str = "RUSTFS_OBS_LOG_EXCLUDE_PATTERNS";
+pub const ENV_OBS_LOG_DELETE_EMPTY_FILES: &str = "RUSTFS_OBS_LOG_DELETE_EMPTY_FILES";
+pub const ENV_OBS_LOG_MIN_FILE_AGE_SECONDS: &str = "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS";
+pub const ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS: &str = "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS";
+pub const ENV_OBS_LOG_DRY_RUN: &str = "RUSTFS_OBS_LOG_DRY_RUN";
+pub const ENV_OBS_LOG_MATCH_MODE: &str = "RUSTFS_OBS_LOG_MATCH_MODE";

-/// Log message capacity for async logging
-pub const ENV_OBS_LOG_MESSAGE_CAPA: &str = "RUSTFS_OBS_LOG_MESSAGE_CAPA";
-
-/// Log flush interval in milliseconds for async logging
-pub const ENV_OBS_LOG_FLUSH_MS: &str = "RUSTFS_OBS_LOG_FLUSH_MS";
-
-/// Default values for log pool
-pub const DEFAULT_OBS_LOG_POOL_CAPA: usize = 10240;
-
-/// Default values for message capacity
-pub const DEFAULT_OBS_LOG_MESSAGE_CAPA: usize = 32768;
-
-/// Default values for flush interval in milliseconds
-pub const DEFAULT_OBS_LOG_FLUSH_MS: u64 = 200;
+/// Default values for log cleanup
+pub const DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES: u64 = 2 * 1024 * 1024 * 1024; // 2 GiB
+pub const DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: u64 = 0; // No single file limit
+pub const DEFAULT_OBS_LOG_COMPRESS_OLD_FILES: bool = true;
+pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL: u32 = 6;
+pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_EXTENSION: &str = "gz";
+pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION: &str = concat!(".", DEFAULT_OBS_LOG_GZIP_COMPRESSION_EXTENSION);
+pub const DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS: u64 = 30;
+pub const DEFAULT_OBS_LOG_DELETE_EMPTY_FILES: bool = true;
+pub const DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS: u64 = 3600; // 1 hour
+pub const DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS: u64 = 1800; // 0.5 hours
+pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false;
+pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix";

 /// Default values for observability configuration
 // ### Supported Environment Values
@@ -87,6 +103,24 @@ mod tests {
        assert_eq!(ENV_OBS_LOG_ROTATION_SIZE_MB, "RUSTFS_OBS_LOG_ROTATION_SIZE_MB");
        assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
        assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
+        assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED");
+        assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
+        assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
+        // Test log cleanup related env keys
+        assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
+        assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
+        assert_eq!(ENV_OBS_LOG_COMPRESS_OLD_FILES, "RUSTFS_OBS_LOG_COMPRESS_OLD_FILES");
+        assert_eq!(ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, "RUSTFS_OBS_LOG_GZIP_COMPRESSION_LEVEL");
+        assert_eq!(
+            ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
+            "RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS"
+        );
+        assert_eq!(ENV_OBS_LOG_EXCLUDE_PATTERNS, "RUSTFS_OBS_LOG_EXCLUDE_PATTERNS");
+        assert_eq!(ENV_OBS_LOG_DELETE_EMPTY_FILES, "RUSTFS_OBS_LOG_DELETE_EMPTY_FILES");
+        assert_eq!(ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS");
+        assert_eq!(ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS");
+        assert_eq!(ENV_OBS_LOG_DRY_RUN, "RUSTFS_OBS_LOG_DRY_RUN");
+        assert_eq!(ENV_OBS_LOG_MATCH_MODE, "RUSTFS_OBS_LOG_MATCH_MODE");
    }

    #[test]
@@ -95,5 +129,6 @@ mod tests {
        assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
        assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
        assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
+        assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix");
    }
 }
@@ -13,7 +13,7 @@
 // limitations under the License.

 use crate::{DEFAULT_SECRET_KEY, ENV_RPC_SECRET, IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
-use rand::{Rng, RngCore};
+use rand::{Rng, RngExt};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::collections::HashMap;
@@ -58,7 +58,7 @@ fn encrypt<T: aes_gcm::aead::Aead>(
    use crate::error::Error;
    use aes_gcm::AeadCore;
    use aes_gcm::aead::array::Array;
-    use rand::RngCore;
+    use rand::Rng;

    let mut nonce: Array<u8, <T as AeadCore>::NonceSize> = Array::default();
    rand::rng().fill_bytes(&mut nonce);
@@ -40,11 +40,18 @@ impl ID {
    pub(crate) fn get_key(&self, password: &[u8], salt: &[u8]) -> Result<[u8; 32], crate::Error> {
        let mut key = [0u8; 32];
        match self {
-            ID::Pbkdf2AESGCM => pbkdf2_hmac::<Sha256>(password, salt, 8192, &mut key),
-            _ => {
-                let params = Params::new(64 * 1024, 1, 4, Some(32))?;
-                let argon_2id = Argon2::new(Algorithm::Argon2id, Version::V0x13, params);
-                argon_2id.hash_password_into(password, salt, &mut key)?;
+            ID::Pbkdf2AESGCM => {
+                pbkdf2_hmac::<Sha256>(password, salt, 8192, &mut key);
+            }
+            ID::Argon2idAESGCM | ID::Argon2idChaCHa20Poly1305 => {
+                const ARGON2_MEMORY: u32 = 64 * 1024;
+                const ARGON2_ITERATIONS: u32 = 1;
+                const ARGON2_PARALLELISM: u32 = 4;
+                const ARGON2_OUTPUT_LEN: usize = 32;
+
+                let params = Params::new(ARGON2_MEMORY, ARGON2_ITERATIONS, ARGON2_PARALLELISM, Some(ARGON2_OUTPUT_LEN))?;
+                let argon2 = Argon2::new(Algorithm::Argon2id, Version::V0x13, params);
+                argon2.hash_password_into(password, salt, &mut key)?;
            }
        }

@@ -106,7 +106,7 @@ fn test_encrypt_decrypt_binary_data() -> Result<(), crate::Error> {
 #[test]
 fn test_encrypt_decrypt_unicode_data() -> Result<(), crate::Error> {
    let unicode_strings = [
-        "Hello, 世界! 🌍",
+        "Hello, 世界！🌍",
        "Тест на русском языке",
        "العربية اختبار",
        "🚀🔐💻🌟⭐",
@@ -20,6 +20,12 @@ pub enum Error {
    #[error("invalid encryption algorithm ID: {0}")]
    ErrInvalidAlgID(u8),

+    #[error("invalid input: {0}")]
+    ErrInvalidInput(String),
+
+    #[error("invalid key length")]
+    ErrInvalidKeyLength,
+
    #[cfg(any(test, feature = "crypto"))]
    #[error("{0}")]
    ErrInvalidLength(#[from] sha2::digest::InvalidLength),
@@ -38,4 +44,13 @@ pub enum Error {

    #[error("jwt err: {0}")]
    ErrJwt(#[from] jsonwebtoken::errors::Error),
+
+    #[error("io error: {0}")]
+    ErrIo(#[from] std::io::Error),
+
+    #[error("invalid signature")]
+    ErrInvalidSignature,
+
+    #[error("invalid token")]
+    ErrInvalidToken,
 }
@@ -23,6 +23,10 @@ rust-version.workspace = true
 [lints]
 workspace = true

+[features]
+default = []
+ftps = []
+
 [dependencies]
 rustfs-ecstore.workspace = true
 rustfs-common.workspace = true
@@ -52,8 +56,7 @@ rand = { workspace = true }
 chrono = { workspace = true }
 md5 = { workspace = true }
 sha2 = { workspace = true }
-suppaftp.workspace = true
+suppaftp = { workspace = true, features = ["tokio", "rustls-aws-lc-rs"] }
 rcgen.workspace = true
 anyhow.workspace = true
-rustls.workspace = true
-rustls-pemfile.workspace = true
+rustls.workspace = true
@@ -0,0 +1,172 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression tests for Issue #2036
+//! Verifies that anonymous access works correctly with bucket policies
+//! when PublicAccessBlock configuration is missing or explicitly set.
+
+use crate::common::{RustFSTestEnvironment, init_logging};
+use aws_sdk_s3::types::PublicAccessBlockConfiguration;
+use serial_test::serial;
+use tracing::info;
+
+async fn setup_public_bucket(
+    env: &RustFSTestEnvironment,
+    bucket_name: &str,
+) -> Result<aws_sdk_s3::Client, Box<dyn std::error::Error + Send + Sync>> {
+    let admin_client = env.create_s3_client();
+
+    admin_client.create_bucket().bucket(bucket_name).send().await?;
+
+    let policy_json = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Sid": "AllowAnonymousGetObject",
+                "Effect": "Allow",
+                "Principal": "*",
+                "Action": ["s3:GetObject"],
+                "Resource": [format!("arn:aws:s3:::{}/*", bucket_name)]
+            }
+        ]
+    })
+    .to_string();
+
+    admin_client
+        .put_bucket_policy()
+        .bucket(bucket_name)
+        .policy(&policy_json)
+        .send()
+        .await?;
+
+    admin_client
+        .put_object()
+        .bucket(bucket_name)
+        .key("test.txt")
+        .body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello anonymous"))
+        .send()
+        .await?;
+
+    Ok(admin_client)
+}
+
+async fn anonymous_get_object(
+    env: &RustFSTestEnvironment,
+    bucket_name: &str,
+    key: &str,
+) -> Result<reqwest::Response, reqwest::Error> {
+    let url = format!("{}/{}/{}", env.url, bucket_name, key);
+    reqwest::Client::new().get(&url).send().await
+}
+
+/// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it
+/// and no PublicAccessBlock configuration exists (ConfigNotFound).
+#[tokio::test]
+#[serial]
+async fn test_anonymous_access_allowed_when_public_access_block_missing() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
+{
+    init_logging();
+    info!("Starting test: anonymous access with missing PublicAccessBlock config...");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket_name = "anon-test-no-pab";
+    let admin_client = setup_public_bucket(&env, bucket_name).await?;
+
+    let _ = admin_client.delete_public_access_block().bucket(bucket_name).send().await;
+
+    let resp = anonymous_get_object(&env, bucket_name, "test.txt").await?;
+    assert_eq!(
+        resp.status().as_u16(),
+        200,
+        "Anonymous GetObject should succeed when no PublicAccessBlock config exists"
+    );
+
+    info!("Test passed: anonymous access allowed with missing PublicAccessBlock config");
+    Ok(())
+}
+
+/// Anonymous GetObject should be denied when RestrictPublicBuckets is true.
+#[tokio::test]
+#[serial]
+async fn test_anonymous_access_denied_when_restrict_public_buckets_enabled()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    info!("Starting test: anonymous access denied with RestrictPublicBuckets=true...");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket_name = "anon-test-restrict";
+    let admin_client = setup_public_bucket(&env, bucket_name).await?;
+
+    admin_client
+        .put_public_access_block()
+        .bucket(bucket_name)
+        .public_access_block_configuration(
+            PublicAccessBlockConfiguration::builder()
+                .restrict_public_buckets(true)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let resp = anonymous_get_object(&env, bucket_name, "test.txt").await?;
+    assert_eq!(
+        resp.status().as_u16(),
+        403,
+        "Anonymous GetObject should be denied when RestrictPublicBuckets is true"
+    );
+
+    info!("Test passed: anonymous access denied with RestrictPublicBuckets=true");
+    Ok(())
+}
+
+/// Anonymous GetObject should succeed when PublicAccessBlock exists but
+/// RestrictPublicBuckets is explicitly false.
+#[tokio::test]
+#[serial]
+async fn test_anonymous_access_allowed_when_restrict_public_buckets_disabled()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    info!("Starting test: anonymous access allowed with RestrictPublicBuckets=false...");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket_name = "anon-test-allow";
+    let admin_client = setup_public_bucket(&env, bucket_name).await?;
+
+    admin_client
+        .put_public_access_block()
+        .bucket(bucket_name)
+        .public_access_block_configuration(
+            PublicAccessBlockConfiguration::builder()
+                .restrict_public_buckets(false)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let resp = anonymous_get_object(&env, bucket_name, "test.txt").await?;
+    assert_eq!(
+        resp.status().as_u16(),
+        200,
+        "Anonymous GetObject should succeed when RestrictPublicBuckets is false"
+    );
+
+    info!("Test passed: anonymous access allowed with RestrictPublicBuckets=false");
+    Ok(())
+}
@@ -0,0 +1,241 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::RustFSTestClusterEnvironment;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::SdkError;
+use bytes::Bytes;
+use serial_test::serial;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+use tracing::{info, warn};
+
+const BUCKET: &str = "conditional-put-race-bucket";
+
+async fn cleanup_object(client: &Client, key: &str) {
+    if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
+        warn!("Failed to delete object '{}' from bucket '{}' during cleanup: {:?}", key, BUCKET, e);
+    }
+}
+
+async fn conditional_put(
+    client: &Client,
+    key: &str,
+    data: &[u8],
+    client_id: usize,
+) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
+    let result = client
+        .put_object()
+        .bucket(BUCKET)
+        .key(key)
+        .body(Bytes::copy_from_slice(data).into())
+        .if_none_match("*")
+        .send()
+        .await;
+
+    match result {
+        Ok(resp) => {
+            info!("  Client {} SUCCEEDED - ETag: {}", client_id, resp.e_tag().unwrap_or("none"));
+            Ok(true)
+        }
+        Err(SdkError::ServiceError(e)) => {
+            let code = e.err().meta().code().unwrap_or("");
+            let message = e.err().meta().message().unwrap_or("");
+            if code == "PreconditionFailed" {
+                warn!("  Client {} got 412 PreconditionFailed", client_id);
+                Ok(false)
+            } else {
+                warn!("  Client {} got ServiceError: code={}, message={}", client_id, code, message);
+                Err(e.into_err().into())
+            }
+        }
+        Err(e) => {
+            warn!("  Client {} got non-ServiceError: {:?}", client_id, e);
+            Err(e.into())
+        }
+    }
+}
+
+async fn run_race_iteration(
+    clients: &[Client],
+    test_key: &str,
+    iteration: usize,
+) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
+    cleanup_object(&clients[0], test_key).await;
+    tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+
+    let head_result = clients[0].head_object().bucket(BUCKET).key(test_key).send().await;
+
+    if head_result.is_ok() {
+        warn!("Warning: Object still exists after cleanup, skipping iteration {}", iteration);
+        return Ok(0);
+    }
+
+    info!("\n=== Iteration {} ===", iteration);
+    info!("Launching {} concurrent conditional PUTs to different nodes...", clients.len());
+
+    let barrier = Arc::new(Barrier::new(clients.len()));
+    let test_key = test_key.to_string();
+
+    let mut handles = vec![];
+    for (i, client) in clients.iter().enumerate() {
+        let client = client.clone();
+        let barrier = barrier.clone();
+        let key = test_key.clone();
+        let data = format!("data from client {}", i).into_bytes();
+
+        let handle = tokio::spawn(async move {
+            barrier.wait().await;
+            conditional_put(&client, &key, &data, i).await
+        });
+        handles.push(handle);
+    }
+
+    let mut success_count = 0;
+    let mut had_error = false;
+    for handle in handles {
+        match handle.await {
+            Ok(Ok(true)) => success_count += 1,
+            Ok(Ok(false)) => {}
+            Ok(Err(e)) => {
+                had_error = true;
+                info!("  Error: {}", e);
+            }
+            Err(e) => {
+                had_error = true;
+                info!("  Task error: {}", e);
+            }
+        }
+    }
+
+    info!("Result: {} out of {} succeeded", success_count, clients.len());
+
+    if success_count > 1 {
+        info!(">>> RACE CONDITION DETECTED!");
+    } else if success_count == 1 {
+        info!(">>> Correct behavior: exactly 1 writer succeeded.");
+    } else if had_error {
+        return Err("all conditional PUTs failed (e.g. cluster/bucket not ready)".into());
+    } else {
+        info!(">>> Unexpected: no writers succeeded.");
+    }
+
+    Ok(success_count)
+}
+
+#[tokio::test]
+#[serial]
+async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    crate::common::init_logging();
+    info!("Starting conditional PUT race test with auto cluster");
+
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    cluster.start().await?;
+
+    cluster.create_test_bucket(BUCKET).await?;
+
+    let clients = cluster.create_all_clients()?;
+
+    let iterations = 5;
+    let mut races_detected = 0;
+    let mut correct_count = 0;
+    let mut error_count = 0;
+
+    for i in 1..=iterations {
+        let test_key = format!("race-test-{}-{}", std::process::id(), i);
+
+        match run_race_iteration(&clients, &test_key, i).await {
+            Ok(success_count) => {
+                if success_count > 1 {
+                    races_detected += 1;
+                } else if success_count == 1 {
+                    correct_count += 1;
+                }
+            }
+            Err(e) => {
+                error_count += 1;
+                warn!("Iteration {} failed (not a race; e.g. cluster/network): {}", i, e)
+            }
+        }
+
+        cleanup_object(&clients[0], &test_key).await;
+        tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
+    }
+
+    info!("\n====================================");
+    info!("SUMMARY");
+    info!("====================================");
+    info!("Total iterations:   {}", iterations);
+    info!("Correct (1 winner): {}", correct_count);
+    info!("Race conditions:    {}", races_detected);
+    info!("Errors (skipped):   {}", error_count);
+
+    assert_eq!(races_detected, 0, "Race conditions detected: {}/{}", races_detected, iterations);
+    assert_eq!(
+        error_count, 0,
+        "{} iteration(s) failed due to errors (e.g. cluster not ready)",
+        error_count
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    crate::common::init_logging();
+    info!("Starting basic conditional PUT test with auto cluster");
+
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    cluster.start().await?;
+
+    cluster.create_test_bucket(BUCKET).await?;
+
+    let client = cluster.create_s3_client(0)?;
+    let test_key = "basic-conditional-put";
+    cleanup_object(&client, test_key).await;
+
+    let result = client
+        .put_object()
+        .bucket(BUCKET)
+        .key(test_key)
+        .body(Bytes::from("first write").into())
+        .if_none_match("*")
+        .send()
+        .await;
+    assert!(result.is_ok(), "First PUT with If-None-Match:* should succeed");
+
+    let result = client
+        .put_object()
+        .bucket(BUCKET)
+        .key(test_key)
+        .body(Bytes::from("second write").into())
+        .if_none_match("*")
+        .send()
+        .await;
+    assert!(result.is_err(), "Second PUT with If-None-Match:* should fail");
+
+    assert!(
+        matches!(result, Err(SdkError::ServiceError(_))),
+        "Expected ServiceError but got different error type"
+    );
+
+    if let Err(SdkError::ServiceError(e)) = result {
+        let code = e.err().meta().code().unwrap_or("");
+        assert_eq!(code, "PreconditionFailed");
+    }
+
+    cleanup_object(&client, test_key).await;
+    Ok(())
+}
@@ -81,6 +81,14 @@ fn build_rustfs_binary() {
    let mut cmd = Command::new("cargo");
    cmd.current_dir(&workspace).args(["build", "--bin", "rustfs"]);

+    // Read features from environment variable for e2e tests
+    if let Ok(features) = std::env::var("RUSTFS_BUILD_FEATURES")
+        && !features.is_empty()
+    {
+        cmd.arg("--features").arg(&features);
+        info!("Building with features: {}", features);
+    }
+
    if !cfg!(debug_assertions) {
        cmd.arg("--release");
    }
@@ -374,3 +382,276 @@ pub async fn awscurl_delete(
 ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
    execute_awscurl(url, "DELETE", None, access_key, secret_key).await
 }
+
+/// Represents a single RustFS server instance in a test cluster.
+///
+/// Each `ClusterNode` tracks the node's network address, base URL for
+/// S3-compatible requests, on-disk data directory, and the underlying
+/// child process handle when the node is running.
+pub struct ClusterNode {
+    pub address: String,
+    pub url: String,
+    pub data_dir: String,
+    pub process: Option<Child>,
+}
+
+/// Test environment for managing a multi-node RustFS cluster.
+///
+/// `RustFSTestClusterEnvironment` is responsible for starting and stopping
+/// a group of `ClusterNode`s, managing their temporary storage directory,
+/// and providing the shared access and secret keys used by tests to
+/// interact with the cluster.
+pub struct RustFSTestClusterEnvironment {
+    pub nodes: Vec<ClusterNode>,
+    pub temp_dir: String,
+    pub access_key: String,
+    pub secret_key: String,
+}
+
+impl RustFSTestClusterEnvironment {
+    /// Create a new RustFS test cluster environment with the specified number of nodes.
+    ///
+    /// Generates a unique temporary root directory for the cluster, allocates an available TCP port
+    /// for each node, creates an independent data directory for every node, and initializes basic
+    /// cluster node configurations (node processes are not started at this stage).
+    ///
+    /// # Arguments
+    ///
+    /// * `node_count` - The number of nodes to create in the cluster, must be a positive integer
+    ///   (an empty cluster will cause errors in subsequent startup operations).
+    ///
+    /// # Returns
+    ///
+    /// * `Ok(Self)` - A new instance of `RustFSTestClusterEnvironment` with initialized node
+    ///   configurations and temporary directory info on success.
+    /// * `Err(Box<dyn Error + Send + Sync>)` - An error if any step fails, such as temporary
+    ///   directory creation failure or available port lookup failure.
+    pub async fn new(node_count: usize) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
+        if node_count == 0 {
+            return Err("Node count must be greater than zero".into());
+        }
+        let temp_dir = format!("/tmp/rustfs_cluster_test_{}", Uuid::new_v4());
+        fs::create_dir_all(&temp_dir).await?;
+
+        let mut nodes = Vec::with_capacity(node_count);
+        for i in 0..node_count {
+            let port = RustFSTestEnvironment::find_available_port().await?;
+            let address = format!("127.0.0.1:{}", port);
+            let url = format!("http://{}", address);
+            let data_dir = format!("{}/node{}", temp_dir, i);
+            fs::create_dir_all(&data_dir).await?;
+
+            nodes.push(ClusterNode {
+                address,
+                url,
+                data_dir,
+                process: None,
+            });
+        }
+
+        Ok(Self {
+            nodes,
+            temp_dir,
+            access_key: DEFAULT_ACCESS_KEY.to_string(),
+            secret_key: DEFAULT_SECRET_KEY.to_string(),
+        })
+    }
+
+    /// Build the volumes argument string for RustFS binary (internal helper method).
+    ///
+    /// Concatenates the address and data directory of all cluster nodes into a single string
+    /// used as the `RUSTFS_VOLUMES` environment variable for RustFS node processes.
+    fn build_volumes_arg(&self) -> String {
+        self.nodes
+            .iter()
+            .map(|n| format!("http://{}{}", n.address, n.data_dir))
+            .collect::<Vec<_>>()
+            .join(" ")
+    }
+
+    /// Start all node processes in the RustFS cluster and wait for the cluster service to be ready.
+    ///
+    /// Spawns a RustFS binary process for each node with necessary environment variable configurations,
+    /// first waits for each node's TCP port to be reachable, then verifies the cluster's S3-compatible
+    /// service availability via the S3 API.
+    ///
+    /// # Returns
+    ///
+    /// * `Ok(())` - All nodes start successfully and the cluster S3 service is ready for requests.
+    /// * `Err(Box<dyn Error + Send + Sync>)` - An error if process spawning fails, TCP port readiness
+    ///   times out, or cluster service readiness times out.
+    pub async fn start(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let binary_path = rustfs_binary_path();
+        let volumes_arg = self.build_volumes_arg();
+
+        for (i, node) in self.nodes.iter_mut().enumerate() {
+            info!("Starting cluster node {} on {}", i, node.address);
+
+            let process = Command::new(&binary_path)
+                .env("RUSTFS_VOLUMES", &volumes_arg)
+                .env("RUSTFS_ADDRESS", &node.address)
+                .env("RUSTFS_ACCESS_KEY", &self.access_key)
+                .env("RUSTFS_SECRET_KEY", &self.secret_key)
+                .env("RUSTFS_CONSOLE_ENABLE", "false")
+                .current_dir(&node.data_dir)
+                .spawn()?;
+
+            node.process = Some(process);
+        }
+
+        for (i, node) in self.nodes.iter().enumerate() {
+            self.wait_for_node_ready(&node.address, i).await?;
+        }
+
+        self.wait_for_service_ready().await?;
+
+        Ok(())
+    }
+
+    /// Wait for a single cluster node's TCP port to become reachable (internal helper method).
+    ///
+    /// Attempts to establish a TCP connection to the node's address, retries up to 60 times
+    /// with a 1-second interval between attempts. Fails if the port is unreachable after all retries.
+    async fn wait_for_node_ready(&self, address: &str, idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        for attempt in 0..60 {
+            if TcpStream::connect(address).await.is_ok() {
+                info!("Node {} ({}) TCP ready after {} attempts", idx, address, attempt + 1);
+                return Ok(());
+            }
+            sleep(Duration::from_secs(1)).await;
+        }
+        Err(format!("Node {} failed to become ready", idx).into())
+    }
+
+    /// Wait for the entire cluster's S3-compatible service to be ready (internal helper method).
+    ///
+    /// Verifies service availability by calling the S3 `list_buckets` API, retries up to 120 times
+    /// with a 1-second interval between attempts. Fails if the API call remains unsuccessful after all retries.
+    async fn wait_for_service_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let client = self.create_s3_client(0)?;
+
+        for attempt in 0..120 {
+            match client.list_buckets().send().await {
+                Ok(_) => {
+                    info!("Cluster service ready after {} attempts", attempt + 1);
+                    return Ok(());
+                }
+                Err(_) => {
+                    sleep(Duration::from_secs(1)).await;
+                }
+            }
+        }
+        Err("Cluster service failed to become ready".into())
+    }
+
+    /// Create an S3 client configured to communicate with a specific cluster node.
+    ///
+    /// Configures the S3 client with the cluster's authentication credentials, a fixed `us-east-1` region,
+    /// the target node's endpoint URL, and enforces path-style access (required for RustFS S3 compatibility).
+    /// Performs a validity check on the node index before creating the client to avoid out-of-bounds errors.
+    ///
+    /// # Arguments
+    ///
+    /// * `node_idx` - The zero-based index of the target cluster node. Must be in the range `[0, total_nodes - 1]`.
+    ///
+    /// # Returns
+    ///
+    /// * `Ok(Client)` - A fully configured AWS S3 `Client` instance for the specified node on success.
+    /// * `Err(Box<dyn Error + Send + Sync>)` - An error if the node index is invalid, or if the S3 client configuration fails.
+    pub fn create_s3_client(&self, node_idx: usize) -> Result<Client, Box<dyn std::error::Error + Send + Sync>> {
+        if node_idx >= self.nodes.len() {
+            return Err("node_idx is invalid".into());
+        }
+        let credentials = Credentials::new(&self.access_key, &self.secret_key, None, None, "cluster-test");
+        let config = Config::builder()
+            .credentials_provider(credentials)
+            .region(Region::new("us-east-1"))
+            .endpoint_url(&self.nodes[node_idx].url)
+            .force_path_style(true)
+            .behavior_version_latest()
+            .build();
+        Ok(Client::from_conf(config))
+    }
+
+    /// Create S3 clients for all nodes in the RustFS cluster and collect them into a vector.
+    ///
+    /// Iterates over all cluster node indices, calls `create_s3_client` for each index, and aggregates
+    /// the resulting clients into a pre-allocated vector. Terminates immediately and returns an error
+    /// if any single node's S3 client creation fails (fails fast behavior).
+    ///
+    /// # Returns
+    ///
+    /// * `Ok(Vec<Client>)` - A vector of configured S3 `Client` instances (one per cluster node) on full success.
+    /// * `Err(Box<dyn Error + Send + Sync>)` - An error with a descriptive message if any client creation fails,
+    ///   including the underlying error from `create_s3_client`.
+    pub fn create_all_clients(&self) -> Result<Vec<Client>, Box<dyn std::error::Error + Send + Sync>> {
+        (0..self.nodes.len()).map(|i| self.create_s3_client(i)).try_fold(
+            Vec::with_capacity(self.nodes.len()),
+            |mut clients, result| match result {
+                Ok(client) => {
+                    clients.push(client);
+                    Ok(clients)
+                }
+                Err(e) => Err(format!("Failed to create S3 client for node: {}", e).into()),
+            },
+        )
+    }
+
+    /// Create a test S3 bucket in the RustFS cluster.
+    ///
+    /// Uses the S3 client of the first cluster node to call the S3 `create_bucket` API and
+    /// create a bucket with the specified name (follows S3 bucket naming conventions).
+    ///
+    /// # Arguments
+    ///
+    /// * `bucket_name` - The name of the bucket to create, must comply with S3 bucket naming
+    ///   rules (lowercase, no spaces, valid characters only).
+    ///
+    /// # Returns
+    ///
+    /// * `Ok(())` - The test bucket is created successfully via the S3 API.
+    /// * `Err(Box<dyn Error + Send + Sync>)` - An error if the S3 `create_bucket` API call fails,
+    ///   such as invalid bucket name, insufficient permissions, or an unready cluster.
+    pub async fn create_test_bucket(&self, bucket_name: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let client = self.create_s3_client(0)?;
+        client.create_bucket().bucket(bucket_name).send().await?;
+        info!("Created test bucket: {}", bucket_name);
+        Ok(())
+    }
+
+    /// Stop all running node processes in the RustFS cluster.
+    ///
+    /// Iterates over all cluster nodes, attempts to kill the spawned RustFS process (if running),
+    /// and waits for the process to exit. Logs an error if process termination or waiting fails,
+    /// but does not panic (fails gracefully).
+    ///
+    /// This method is automatically called by the `Drop` trait when the cluster environment
+    /// is destroyed, and can also be called manually to stop the cluster early.
+    pub fn stop(&mut self) {
+        for (i, node) in self.nodes.iter_mut().enumerate() {
+            if let Some(mut process) = node.process.take() {
+                info!("Stopping cluster node {}", i);
+                if let Err(e) = process.kill() {
+                    error!("Failed to kill cluster node {}: {}", i, e);
+                }
+                if let Err(e) = process.wait() {
+                    error!("Failed to wait for cluster node {} to exit: {}", i, e);
+                }
+            }
+        }
+    }
+}
+
+impl Drop for RustFSTestClusterEnvironment {
+    /// Clean up the RustFS test cluster environment when the instance is dropped.
+    ///
+    /// Automatically calls the `stop` method to terminate all running node processes, then
+    /// attempts to delete the cluster's temporary root directory and all its contents.
+    /// Logs a warning if directory deletion fails (does not affect program exit).
+    fn drop(&mut self) {
+        self.stop();
+        if let Err(e) = std::fs::remove_dir_all(&self.temp_dir) {
+            warn!("Failed to clean up cluster temp directory {}: {}", self.temp_dir, e);
+        }
+    }
+}
@@ -82,4 +82,60 @@ mod tests {

        env.stop_server();
    }
+
+    /// Issue #1857: Content-Encoding "aws-chunked" is used by SigV4 streaming clients and must
+    /// not be stored or returned. Upload with aws-chunked and verify GET/HEAD do not return it.
+    #[tokio::test]
+    #[serial]
+    async fn test_content_encoding_aws_chunked_not_returned_issue_1857() {
+        init_logging();
+        info!("Issue #1857: aws-chunked must not be persisted or returned");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "content-encoding-aws-chunked-test";
+        let key = "streamed/object.bin";
+        let content = b"streaming upload body";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .content_encoding("aws-chunked")
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("PUT failed");
+
+        let get_resp = client.get_object().bucket(bucket).key(key).send().await.expect("GET failed");
+        assert_eq!(
+            get_resp.content_encoding(),
+            None,
+            "GET must not return Content-Encoding when only aws-chunked was sent (Issue #1857)"
+        );
+
+        let head_resp = client
+            .head_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("HEAD failed");
+        assert_eq!(
+            head_resp.content_encoding(),
+            None,
+            "HEAD must not return Content-Encoding when only aws-chunked was sent (Issue #1857)"
+        );
+
+        env.stop_server();
+    }
 }
@@ -0,0 +1,288 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for Issue #1878:
+//! "In a versioned Bucket, DeleteMarkers are not appearing straight after
+//! a delete_objects is called."
+//!
+//! Root cause: `delete_versions_internal` wrote new xl.meta to disk via
+//! `write_all_private` without invalidating the `GlobalFileCache`. Subsequent
+//! calls to `read_metadata` returned the stale cached xl.meta (without the
+//! delete marker), making `list_object_versions` show the old version as
+//! `IsLatest=true` rather than the new delete marker.
+//!
+//! Fix: `write_all_private` now calls `get_global_file_cache().invalidate()`
+//! after every successful write, and `rename_data` also invalidates the cache
+//! for the destination path after the atomic rename.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
+    use serial_test::serial;
+    use tracing::info;
+
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    /// Regression test for Issue #1878.
+    ///
+    /// Verifies that after calling `delete_objects` (the batch plural API) on
+    /// a versioned bucket, calling `list_object_versions` **immediately** (with
+    /// no sleep) returns the newly-created DeleteMarker with `is_latest = true`.
+    #[tokio::test]
+    #[serial]
+    async fn test_delete_objects_delete_marker_immediately_visible() {
+        init_logging();
+        info!("🧪 TEST: DeleteMarker from delete_objects is immediately visible via list_object_versions");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-delete-marker-visibility";
+        let key = "test-prefix/test-object.txt";
+
+        // 1. Create bucket
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+        info!("✅ Bucket created: {}", bucket);
+
+        // 2. Enable versioning
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to enable versioning");
+        info!("✅ Versioning enabled");
+
+        // 3. Put an object
+        let put_resp = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello versioned world\n"))
+            .send()
+            .await
+            .expect("Failed to put object");
+        let original_version_id = put_resp.version_id().map(str::to_string);
+        info!("✅ Object put, version_id: {:?}", original_version_id);
+        assert!(
+            original_version_id.is_some(),
+            "PutObject should return a version_id in a versioned bucket"
+        );
+
+        // 4. List versions – should show exactly 1 Version with is_latest = true
+        let list_before = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions before delete");
+
+        let versions_before = list_before.versions().to_vec();
+        let markers_before = list_before.delete_markers().to_vec();
+        assert_eq!(versions_before.len(), 1, "Should have 1 version before delete");
+        assert!(markers_before.is_empty(), "Should have no delete markers before delete");
+        assert_eq!(
+            versions_before[0].is_latest(),
+            Some(true),
+            "Original version should be latest before delete"
+        );
+        info!("✅ Verified: 1 version, 0 delete markers before delete_objects");
+
+        // 5. Call delete_objects (plural, no version_id → creates a delete marker)
+        let del_resp = client
+            .delete_objects()
+            .bucket(bucket)
+            .delete(
+                Delete::builder()
+                    .objects(
+                        ObjectIdentifier::builder()
+                            .key(key)
+                            .build()
+                            .expect("Failed to build object identifier"),
+                    )
+                    .build()
+                    .expect("Failed to build delete request"),
+            )
+            .send()
+            .await
+            .expect("Failed to call delete_objects");
+
+        let deleted = del_resp.deleted();
+        assert_eq!(deleted.len(), 1, "delete_objects should report 1 deleted object");
+        let delete_marker_version_id = deleted[0].delete_marker_version_id().map(str::to_string);
+        assert!(
+            deleted[0].delete_marker() == Some(true),
+            "delete_objects should report delete_marker=true for versioned delete"
+        );
+        assert!(
+            delete_marker_version_id.is_some(),
+            "delete_objects should return a delete_marker_version_id"
+        );
+        info!("✅ delete_objects succeeded, delete_marker_version_id: {:?}", delete_marker_version_id);
+
+        // 6. List versions IMMEDIATELY (no sleep!) – this is the key regression assertion.
+        //    Before the fix, the stale file cache caused the listing to return the original
+        //    version as is_latest=true without any delete marker.
+        let list_after = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions immediately after delete_objects");
+
+        let versions_after = list_after.versions().to_vec();
+        let markers_after = list_after.delete_markers().to_vec();
+
+        info!("Versions after delete_objects: {:?}", versions_after.len());
+        info!("Delete markers after delete_objects: {:?}", markers_after.len());
+
+        // The delete marker must be visible immediately – no delay required.
+        assert_eq!(
+            markers_after.len(),
+            1,
+            "list_object_versions should return exactly 1 delete marker IMMEDIATELY after delete_objects (regression #1878)"
+        );
+        assert_eq!(markers_after[0].is_latest(), Some(true), "The delete marker should be is_latest=true");
+        assert_eq!(
+            markers_after[0].version_id().map(str::to_string),
+            delete_marker_version_id,
+            "The delete marker version_id should match the one returned by delete_objects"
+        );
+
+        // The original version should still be present, but is_latest = false
+        assert_eq!(versions_after.len(), 1, "The original version should still be present");
+        assert_eq!(
+            versions_after[0].is_latest(),
+            Some(false),
+            "Original version should be is_latest=false after delete marker is created"
+        );
+
+        info!("✅ TEST PASSED: Delete marker is immediately visible after delete_objects (no sleep needed)");
+    }
+
+    /// Additional regression check: verify that multiple objects deleted via
+    /// a single `delete_objects` call all have their delete markers visible
+    /// immediately afterwards.
+    #[tokio::test]
+    #[serial]
+    async fn test_delete_objects_multiple_keys_delete_markers_immediately_visible() {
+        init_logging();
+        info!("🧪 TEST: Multiple delete markers from delete_objects are immediately visible");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-multi-delete-marker-visibility";
+        let keys = ["obj1.txt", "obj2.txt", "obj3.txt"];
+
+        // Create bucket and enable versioning
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to enable versioning");
+
+        // Put all objects
+        for key in &keys {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(*key)
+                .body(aws_sdk_s3::primitives::ByteStream::from_static(b"data"))
+                .send()
+                .await
+                .unwrap_or_else(|_| panic!("Failed to put object {}", key));
+        }
+        info!("✅ Put {} objects", keys.len());
+
+        // Delete all objects with a single delete_objects call
+        let identifiers: Vec<ObjectIdentifier> = keys
+            .iter()
+            .map(|k| {
+                ObjectIdentifier::builder()
+                    .key(*k)
+                    .build()
+                    .expect("Failed to build ObjectIdentifier")
+            })
+            .collect();
+
+        let del_resp = client
+            .delete_objects()
+            .bucket(bucket)
+            .delete(
+                Delete::builder()
+                    .set_objects(Some(identifiers))
+                    .build()
+                    .expect("Failed to build delete"),
+            )
+            .send()
+            .await
+            .expect("Failed to call delete_objects");
+
+        assert_eq!(del_resp.deleted().len(), keys.len(), "All keys should be reported deleted");
+        info!("✅ delete_objects deleted {} objects", keys.len());
+
+        // Immediately list all versions and check each key has a delete marker
+        let list_resp = client
+            .list_object_versions()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to list object versions");
+
+        let markers = list_resp.delete_markers().to_vec();
+        assert_eq!(
+            markers.len(),
+            keys.len(),
+            "Each deleted key should have an immediately-visible delete marker (regression #1878)"
+        );
+
+        for marker in &markers {
+            assert_eq!(marker.is_latest(), Some(true), "Each delete marker should be is_latest=true");
+        }
+
+        info!("✅ TEST PASSED: All {} delete markers are immediately visible", keys.len());
+    }
+}
@@ -0,0 +1,208 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! E2E tests for group management (fixes #2028).
+
+use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_put, init_logging};
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::{Client, Config};
+use serial_test::serial;
+use tracing::info;
+
+fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-group-test");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+/// Test that deleting a group with members fails, and deleting an empty group succeeds.
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+#[ignore = "requires awscurl and spawns a real RustFS server"]
+async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    // 1. Create a user
+    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
+    let user_body = serde_json::json!({
+        "secretKey": "testuser1secret",
+        "status": "enabled"
+    });
+    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created testuser1");
+
+    // 2. Create a group with testuser1 as a member
+    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let add_member_body = serde_json::json!({
+        "group": "testgroup",
+        "members": ["testuser1"],
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Added testuser1 to testgroup");
+
+    // 3. Attempt to delete the group while it still has members — should fail
+    let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
+    let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
+    assert!(delete_result.is_err(), "deleting a non-empty group should fail");
+    info!("Delete of non-empty group correctly rejected");
+
+    // 4. Remove the member from the group
+    let remove_member_body = serde_json::json!({
+        "group": "testgroup",
+        "members": ["testuser1"],
+        "isRemove": true,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Removed testuser1 from testgroup");
+
+    // 5. Delete the now-empty group — should succeed
+    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    info!("Deleted empty testgroup successfully");
+
+    // 6. Verify the group no longer exists
+    let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
+    let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
+    assert!(get_result.is_err(), "group should no longer exist after deletion");
+    info!("Confirmed testgroup no longer exists");
+
+    Ok(())
+}
+
+/// Test that a user with only group membership (no explicit user policy) gets group policies
+/// and can perform actions allowed by the group (regression test for #2028.1).
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+#[ignore = "requires awscurl and spawns a real RustFS server"]
+async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let user_name = "grouponlyuser";
+    let user_secret = "grouponlysecret";
+    let group_name = "policygroup";
+    let policy_name = "ListBucketsOnlyPolicy";
+
+    // 1. Create canned policy that allows ListAllMyBuckets only
+    let policy_doc = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:ListAllMyBuckets"],
+            "Resource": ["*"]
+        }]
+    });
+    let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created canned policy {}", policy_name);
+
+    // 2. Create user with no explicit policy
+    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
+    let user_body = serde_json::json!({
+        "secretKey": user_secret,
+        "status": "enabled"
+    });
+    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created user {} with no explicit policy", user_name);
+
+    // 3. Add user to group (creates group with this member; user_group_memberships must be updated)
+    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let add_member_body = serde_json::json!({
+        "group": group_name,
+        "members": [user_name],
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Added {} to group {}", user_name, group_name);
+
+    // 4. Attach policy to group
+    let set_policy_url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
+        env.url, policy_name, group_name
+    );
+    awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
+    info!("Attached policy {} to group {}", policy_name, group_name);
+
+    // 5. User with only group (no user policy) should be able to list buckets
+    let user_client = create_user_s3_client(&env, user_name, user_secret);
+    let list_result = user_client.list_buckets().send().await;
+    list_result.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> {
+        format!("User with only group membership should get group policies and list buckets: {}", e).into()
+    })?;
+    info!("User with only group successfully listed buckets (group policies applied)");
+
+    Ok(())
+}
+
+/// Test that after deleting a user who was the only member of a group, the group can be deleted
+/// (regression test for #2028.2: delete group uses backend membership, not stale cache).
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+#[ignore = "requires awscurl and spawns a real RustFS server"]
+async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let user_name = "solemember";
+    let user_secret = "solemembersecret";
+    let group_name = "soledeletegroup";
+
+    // 1. Create user
+    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
+    let user_body = serde_json::json!({
+        "secretKey": user_secret,
+        "status": "enabled"
+    });
+    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created user {}", user_name);
+
+    // 2. Add user to group
+    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let add_member_body = serde_json::json!({
+        "group": group_name,
+        "members": [user_name],
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Added {} to group {}", user_name, group_name);
+
+    // 3. Delete the user (backend and cache update so group membership becomes empty)
+    let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
+    awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
+    info!("Deleted user {}", user_name);
+
+    // 4. Deleting the group should succeed (backend has empty members; no stale cache)
+    let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
+    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    info!("Deleted group {} after user was removed", group_name);
+
+    Ok(())
+}
@@ -117,7 +117,7 @@ pub async fn create_default_key(

 /// Create a KMS key with a specific ID (by directly writing to the key directory)
 pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    use rand::RngCore;
+    use rand::Rng;
    use std::collections::HashMap;
    use tokio::fs;

@@ -29,6 +29,10 @@ mod data_usage_test;
 #[cfg(test)]
 mod kms;

+// Regression test for issue #1797
+#[cfg(test)]
+mod list_objects_duplicates_test;
+
 // Quota tests
 #[cfg(test)]
 mod quota_test;
@@ -36,6 +40,10 @@ mod quota_test;
 #[cfg(test)]
 mod bucket_policy_check_test;

+// Regression tests for Issue #2036: anonymous access with PublicAccessBlock
+#[cfg(test)]
+mod anonymous_access_test;
+
 // Special characters in path test modules
 #[cfg(test)]
 mod special_chars_test;
@@ -54,6 +62,10 @@ mod policy;

 #[cfg(test)]
 mod compression_test;
+
+// Regression test for Issue #1878: DeleteMarkers not visible immediately after delete_objects
+#[cfg(test)]
+mod delete_objects_versioning_test;
 #[cfg(test)]
 mod protocols;

@@ -61,6 +73,13 @@ mod protocols;
 #[cfg(test)]
 mod object_lock;

+#[cfg(test)]
+mod cluster_concurrency_test;
+
 // PutObject / MultipartUpload with checksum (Content-MD5, x-amz-checksum-*)
 #[cfg(test)]
 mod checksum_upload_test;
+
+// Group deletion tests
+#[cfg(test)]
+mod group_delete_test;
@@ -0,0 +1,135 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::primitives::ByteStream;
+    use serial_test::serial;
+    use tracing::info;
+
+    /// Helper function to create an S3 client for testing
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    /// Helper function to create a test bucket
+    async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let mut retries = 20;
+        loop {
+            match client.create_bucket().bucket(bucket).send().await {
+                Ok(_) => {
+                    info!("Bucket {} created successfully", bucket);
+                    return Ok(());
+                }
+                Err(e) => {
+                    // Ignore if bucket already exists
+                    if e.to_string().contains("BucketAlreadyOwnedByYou") || e.to_string().contains("BucketAlreadyExists") {
+                        info!("Bucket {} already exists", bucket);
+                        return Ok(());
+                    }
+                    if retries > 0 {
+                        retries -= 1;
+                        info!("Bucket creation failed, retrying... ({})", retries);
+                        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+                        continue;
+                    }
+                    return Err(Box::new(e));
+                }
+            }
+        }
+    }
+
+    /// Test ensuring that ListObjectsV2 returns unique CommonPrefixes even if "folder" objects exist.
+    ///
+    /// Bug Reference: Issue #1797
+    /// Veeam creates 0-byte objects ending in '/' (e.g. "folder/") to represent folders.
+    /// If "folder/file.txt" also exists, "folder/" is a CommonPrefix.
+    /// The bug was that "folder/" (the object) and "folder/" (derived prefix) were both added to CommonPrefixes
+    /// when delimiter was "/" because the deduplication check was explicitly skipped for "/" delimiter.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_unique_common_prefixes() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 should return unique CommonPrefixes");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-unique-prefixes";
+
+        // Create bucket
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        // 1. Create a file inside a folder
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("folder/file.txt")
+            .body(ByteStream::from_static(b"content"))
+            .send()
+            .await
+            .expect("Failed to create file inside folder");
+
+        // 2. Create the "folder" object itself (Veeam behavior)
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("folder/")
+            .body(ByteStream::from_static(b""))
+            .send()
+            .await
+            .expect("Failed to create folder object");
+
+        // 3. List with delimiter="/"
+        let result = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .delimiter("/")
+            .send()
+            .await
+            .expect("Failed to list objects");
+
+        // Verify prefixes
+        let prefixes = result.common_prefixes();
+        info!("CommonPrefixes: {:?}", prefixes);
+
+        // Should contain "folder/" exactly once
+        let folder_prefixes: Vec<_> = prefixes.iter().filter(|p| p.prefix() == Some("folder/")).collect();
+
+        assert_eq!(
+            folder_prefixes.len(),
+            1,
+            "Expected exactly 1 'folder/' prefix, found {}",
+            folder_prefixes.len()
+        );
+
+        // Verify that "folder/" is NOT returned as an object in Contents.
+        // For this regression test, we expect "folder/" to be represented only as a CommonPrefix
+        // (rolled up from "folder/file.txt" and the explicit "folder/" object), and to appear there
+        // exactly once. It must not appear in Contents at all.
+
+        // Ensure "folder/" is NOT in contents (Contents)
+        let folder_in_contents = result.contents().iter().any(|o| o.key() == Some("folder/"));
+        assert!(
+            !folder_in_contents,
+            "Expected 'folder/' to be rolled up into CommonPrefixes, but found it in Contents"
+        );
+
+        // Stop the RustFS server to ensure proper cleanup
+        env.stop_server();
+    }
+}
@@ -1,6 +1,6 @@
 # Protocol E2E Tests

-FTPS and SFTP protocol end-to-end tests for RustFS.
+FTPS protocol end-to-end tests for RustFS.

 ## Prerequisites

@@ -21,12 +21,7 @@ brew install sshpass openssh

 Run all protocol tests:
 ```bash
-cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
-```
-
-Run only FTPS tests:
-```bash
-cargo test --package e2e_test test_ftps_core_operations -- --test-threads=1 --nocapture
+RUSTFS_BUILD_FEATURES=ftps cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
 ```

 ## Test Coverage
@@ -41,4 +36,5 @@ cargo test --package e2e_test test_ftps_core_operations -- --test-threads=1 --no
 - cd nonexistent bucket (should fail)
 - delete object
 - cdup
- rmdir delete bucket
+- rmdir delete bucket
+
@@ -18,8 +18,7 @@ use crate::common::rustfs_binary_path;
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
 use rcgen::generate_simple_self_signed;
-use rustls::crypto::aws_lc_rs::default_provider;
-use rustls::{ClientConfig, RootCertStore};
+use rustls::{ClientConfig, RootCertStore, pki_types::CertificateDer, pki_types::pem::PemObject};
 use std::io::Cursor;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -36,15 +35,26 @@ const FTPS_ADDRESS: &str = "127.0.0.1:9021";
 pub async fn test_ftps_core_operations() -> Result<()> {
    let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;

-    // Generate and write certificate
-    let cert = generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])?;
-    let cert_path = PathBuf::from(&env.temp_dir).join("ftps.crt");
-    let key_path = PathBuf::from(&env.temp_dir).join("ftps.key");
+    let cert_dir = PathBuf::from(&env.temp_dir).join("ftps_certs");
+    tokio::fs::create_dir_all(&cert_dir).await?;

-    let cert_pem = cert.cert.pem();
-    let key_pem = cert.signing_key.serialize_pem();
-    tokio::fs::write(&cert_path, &cert_pem).await?;
-    tokio::fs::write(&key_path, &key_pem).await?;
+    // Generate default certificate for root directory
+    let default_cert = generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])?;
+    let default_cert_path = cert_dir.join("rustfs_cert.pem");
+    let default_key_path = cert_dir.join("rustfs_key.pem");
+    tokio::fs::write(&default_cert_path, default_cert.cert.pem()).await?;
+    tokio::fs::write(&default_key_path, default_cert.signing_key.serialize_pem()).await?;
+
+    // Create subdirectory for domain-specific certificate
+    let example_domain_dir = cert_dir.join("example1.com");
+    tokio::fs::create_dir_all(&example_domain_dir).await?;
+    let domain_cert = generate_simple_self_signed(vec!["example1.com".to_string()])?;
+    let domain_cert_path = example_domain_dir.join("rustfs_cert.pem");
+    let domain_key_path = example_domain_dir.join("rustfs_key.pem");
+    tokio::fs::write(&domain_cert_path, domain_cert.cert.pem()).await?;
+    tokio::fs::write(&domain_key_path, domain_cert.signing_key.serialize_pem()).await?;
+
+    info!("Generated 2 certificates in {:?}", cert_dir);

    // Start server manually
    info!("Starting FTPS server on {}", FTPS_ADDRESS);
@@ -52,8 +62,7 @@ pub async fn test_ftps_core_operations() -> Result<()> {
    let mut server_process = Command::new(&binary_path)
        .env("RUSTFS_FTPS_ENABLE", "true")
        .env("RUSTFS_FTPS_ADDRESS", FTPS_ADDRESS)
-        .env("RUSTFS_FTPS_CERTS_FILE", cert_path.to_str().unwrap())
-        .env("RUSTFS_FTPS_KEY_FILE", key_path.to_str().unwrap())
+        .env("RUSTFS_FTPS_CERTS_DIR", cert_dir.to_str().unwrap())
        .arg(&env.temp_dir)
        .spawn()?;

@@ -64,8 +73,8 @@ pub async fn test_ftps_core_operations() -> Result<()> {
            .await
            .map_err(|e| anyhow::anyhow!("{}", e))?;

-        // Install the aws-lc-rs crypto provider
-        default_provider()
+        // Build ServerConfig with SNI support
+        rustls::crypto::aws_lc_rs::default_provider()
            .install_default()
            .map_err(|e| anyhow::anyhow!("Failed to install crypto provider: {:?}", e))?;

@@ -73,8 +82,8 @@ pub async fn test_ftps_core_operations() -> Result<()> {
        let mut root_store = RootCertStore::empty();
        // Add the self-signed certificate to the trust store for e2e
        // Note: In a real environment, you'd use proper root certificates
-        let cert_pem = cert.cert.pem();
-        let cert_der = rustls_pemfile::certs(&mut Cursor::new(cert_pem))
+        let cert_pem = default_cert.cert.pem();
+        let cert_der = CertificateDer::pem_reader_iter(&mut Cursor::new(cert_pem))
            .collect::<Result<Vec<_>, _>>()
            .map_err(|e| anyhow::anyhow!("Failed to parse cert: {}", e))?;

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-//! Protocol tests for FTPS and SFTP
+//! Protocol tests for FTPS

 pub mod ftps_core;
 pub mod test_env;
@@ -59,12 +59,9 @@ struct TestDefinition {
 impl ProtocolTestSuite {
    /// Create default test suite
    pub fn new() -> Self {
-        let tests = vec![
-            TestDefinition {
-                name: "test_ftps_core_operations".to_string(),
-            },
-            // TestDefinition { name: "test_sftp_core_operations".to_string() },
-        ];
+        let tests = vec![TestDefinition {
+            name: "test_ftps_core_operations".to_string(),
+        }];

        Self { tests }
    }
@@ -86,10 +83,6 @@ impl ProtocolTestSuite {
                    info!("=== Starting FTPS Module Test ===");
                    "FTPS core operations (put, ls, mkdir, rmdir, delete)"
                }
-                "test_sftp_core_operations" => {
-                    info!("=== Starting SFTP Module Test ===");
-                    "SFTP core operations (put, ls, mkdir, rmdir, delete)"
-                }
                _ => "",
            };

@@ -128,7 +121,6 @@ impl ProtocolTestSuite {
    async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        match test_def.name.as_str() {
            "test_ftps_core_operations" => test_ftps_core_operations().await.map_err(|e| e.into()),
-            // "test_sftp_core_operations" => test_sftp_core_operations().await.map_err(|e| e.into()),
            _ => Err(format!("Test {} not implemented", test_def.name).into()),
        }
    }
@@ -422,7 +422,7 @@ mod integration_tests {
        // Check if we can upload 1KB (should succeed - we haven't used the full quota yet)
        let check_result = env.check_bucket_quota("PUT", 1024).await?;
        assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
-        assert_eq!(check_result.get("remaining_quota").unwrap().as_u64().unwrap(), 512 * 1024 - 1024);
+        assert_eq!(check_result.get("remaining_quota").unwrap().as_u64().unwrap(), 523264); // 511 * 1024

        // Check if we can upload 600KB (should fail - would exceed quota)
        let check_result = env.check_bucket_quota("PUT", 600 * 1024).await?;
@@ -570,6 +570,60 @@ mod integration_tests {
        Ok(())
    }

+    /// Test that a normal user with `readwrite` policy can read quota but cannot set/clear quota.
+    #[tokio::test]
+    #[serial]
+    async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        init_logging();
+        let env = QuotaTestEnv::new().await?;
+        env.create_bucket().await?;
+
+        // Admin sets quota first
+        env.set_bucket_quota(1024 * 1024).await?;
+
+        // Create a normal user via admin API
+        let normal_ak = "normaluser";
+        let normal_sk = "normaluser123";
+        let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.env.url, normal_ak);
+        let user_body = serde_json::json!({ "secretKey": normal_sk, "status": "enabled" }).to_string();
+        awscurl_put(&add_user_url, &user_body, &env.env.access_key, &env.env.secret_key).await?;
+
+        // Attach `readwrite` policy to the normal user
+        let policy_url = format!(
+            "{}/rustfs/admin/v3/set-user-or-group-policy?policyName=readwrite&userOrGroup={}&isGroup=false",
+            env.env.url, normal_ak
+        );
+        awscurl_put(&policy_url, "", &env.env.access_key, &env.env.secret_key).await?;
+
+        // Normal user reads quota — should succeed
+        let get_url = format!("{}/rustfs/admin/v3/quota/{}", env.env.url, env.bucket_name);
+        let resp = awscurl_get(&get_url, normal_ak, normal_sk).await?;
+        let quota_info: serde_json::Value = serde_json::from_str(&resp)?;
+        assert_eq!(quota_info.get("quota").and_then(|v| v.as_u64()), Some(1024 * 1024));
+
+        // Normal user reads quota stats — should succeed
+        let stats_url = format!("{}/rustfs/admin/v3/quota-stats/{}", env.env.url, env.bucket_name);
+        let resp = awscurl_get(&stats_url, normal_ak, normal_sk).await?;
+        assert!(resp.contains("quota_limit"));
+
+        // Normal user sets quota — should be denied
+        let set_resp = awscurl_put(
+            &get_url,
+            &serde_json::json!({"quota": 2048, "quota_type": "HARD"}).to_string(),
+            normal_ak,
+            normal_sk,
+        )
+        .await;
+        assert!(set_resp.is_err(), "normal user should not be able to set quota");
+
+        // Normal user clears quota — should be denied
+        let del_resp = awscurl_delete(&get_url, normal_ak, normal_sk).await;
+        assert!(del_resp.is_err(), "normal user should not be able to clear quota");
+
+        env.cleanup_bucket().await?;
+        Ok(())
+    }
+
    #[tokio::test]
    #[serial]
    async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -67,6 +67,22 @@ mod tests {
        Ok(())
    }

+    async fn suspend_versioning(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let versioning_config = VersioningConfiguration::builder()
+            .status(BucketVersioningStatus::Suspended)
+            .build();
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(versioning_config)
+            .send()
+            .await?;
+
+        info!("✅ Versioning suspended for bucket {}", bucket);
+        Ok(())
+    }
+
    /// Test 1: PutObject should return version_id when versioning is enabled
    /// This directly addresses the Veeam issue from #1066
    #[tokio::test]
@@ -395,4 +411,91 @@ mod tests {

        info!("✅ PASSED: Veeam backup workflow simulation completed successfully");
    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_terraform_put_after_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        // Use a versioned bucket for this test
+        let bucket = "terraform";
+
+        let client = env.create_s3_client();
+        env.create_test_bucket(bucket).await?;
+
+        let key = "terraform.tfstate";
+        let response = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from(b"v1".to_vec()))
+            .send()
+            .await;
+        assert!(response.is_ok());
+
+        client.delete_object().bucket(bucket).key(key).send().await?;
+
+        let response = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from(b"v1".to_vec()))
+            .send()
+            .await;
+
+        assert!(response.is_ok());
+
+        let get_response = client.get_object().bucket(bucket).key(key).send().await;
+        assert!(get_response.is_ok(), "Object should exist after PUT");
+
+        Ok(())
+    }
+
+    /// Test 7: PutObject should return "null" version_id when versioning is Suspended
+    #[tokio::test]
+    #[serial]
+    async fn test_put_object_returns_null_version_id_with_suspended_versioning() {
+        init_logging();
+        info!("🧪 TEST: PutObject returns null version_id with versioning suspended");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-suspended-version-id";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+        suspend_versioning(&client, bucket)
+            .await
+            .expect("Failed to suspend versioning");
+
+        let key = "test-file-suspended.txt";
+        let content = b"Test content for suspended version ID test";
+
+        info!("📤 Uploading object to suspended versioning bucket");
+        let result = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await;
+
+        assert!(result.is_ok(), "PutObject failed: {:?}", result.err());
+        let output = result.unwrap();
+
+        info!("📥 PutObject response - version_id: {:?}", output.version_id);
+
+        // When suspended, version_id must be "null"
+        assert_eq!(
+            output.version_id.as_deref(),
+            Some("null"),
+            "❌ FAILED: version_id should be 'null' when versioning is suspended"
+        );
+
+        info!("✅ PASSED: PutObject correctly returns 'null' version_id");
+    }
 }
@@ -52,6 +52,7 @@ glob = { workspace = true }
 thiserror.workspace = true
 flatbuffers.workspace = true
 futures.workspace = true
+futures-util.workspace = true
 tracing.workspace = true
 serde.workspace = true
 time.workspace = true
@@ -60,6 +61,8 @@ serde_json.workspace = true
 quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
 s3s.workspace = true
 http.workspace = true
+http-body = { workspace = true }
+http-body-util.workspace = true
 url.workspace = true
 uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
 reed-solomon-simd = { workspace = true }
@@ -108,6 +111,8 @@ google-cloud-storage = { workspace = true }
 google-cloud-auth = { workspace = true }
 aws-config = { workspace = true }
 faster-hex = { workspace = true }
+ratelimit = { workspace = true }
+aws-smithy-http-client.workspace = true

 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
@@ -12,6 +12,5 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-//! SFTP protocol implementation
-pub mod handler;
-pub mod server;
+pub mod monitor;
+pub mod reader;
@@ -0,0 +1,321 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::bucket::bandwidth::reader::BucketOptions;
+use ratelimit::{Error as RatelimitError, Ratelimiter};
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex, RwLock};
+use std::time::Duration;
+use tracing::warn;
+
+#[derive(Clone)]
+pub struct BucketThrottle {
+    limiter: Arc<Mutex<Ratelimiter>>,
+    pub node_bandwidth_per_sec: i64,
+}
+
+impl BucketThrottle {
+    fn new(node_bandwidth_per_sec: i64) -> Result<Self, RatelimitError> {
+        let node_bandwidth_per_sec = node_bandwidth_per_sec.max(1);
+        let amount = node_bandwidth_per_sec as u64;
+        let limiter_inner = Ratelimiter::builder(amount, Duration::from_secs(1))
+            .max_tokens(amount)
+            .build()?;
+        Ok(Self {
+            limiter: Arc::new(Mutex::new(limiter_inner)),
+            node_bandwidth_per_sec,
+        })
+    }
+
+    pub fn burst(&self) -> u64 {
+        self.limiter.lock().unwrap_or_else(|e| e.into_inner()).max_tokens()
+    }
+
+    /// The ratelimit crate (0.10.0) does not provide a bulk token consumption API.
+    /// try_wait() first to consume 1 token AND trigger the internal refill
+    /// mechanism (tokens are only refilled during try_wait/wait calls).
+    /// directly adjust available tokens via set_available() to consume the remaining amount.
+    pub(crate) fn consume(&self, n: u64) -> (u64, f64, u64) {
+        let guard = self.limiter.lock().unwrap_or_else(|e| {
+            warn!("bucket throttle mutex poisoned, recovering");
+            e.into_inner()
+        });
+        if n == 0 {
+            return (0, guard.rate(), 0);
+        }
+        let mut consumed = 0u64;
+        if guard.try_wait().is_ok() {
+            consumed = 1;
+        }
+        let available = guard.available();
+        let to_consume = n - consumed;
+        let batch = to_consume.min(available);
+        if batch > 0 {
+            let _ = guard.set_available(available - batch);
+            consumed += batch;
+        }
+        let deficit = n.saturating_sub(consumed);
+        let rate = guard.rate();
+        (deficit, rate, consumed)
+    }
+}
+
+pub struct Monitor {
+    t_lock: RwLock<HashMap<BucketOptions, BucketThrottle>>,
+    pub node_count: u64,
+}
+
+impl Monitor {
+    pub fn new(num_nodes: u64) -> Arc<Self> {
+        let node_cnt = num_nodes.max(1);
+        Arc::new(Monitor {
+            t_lock: RwLock::new(HashMap::new()),
+            node_count: node_cnt,
+        })
+    }
+
+    pub fn delete_bucket(&self, bucket: &str) {
+        self.t_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .retain(|opts, _| opts.name != bucket);
+    }
+
+    pub fn delete_bucket_throttle(&self, bucket: &str, arn: &str) {
+        let opts = BucketOptions {
+            name: bucket.to_string(),
+            replication_arn: arn.to_string(),
+        };
+        self.t_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .remove(&opts);
+    }
+
+    pub fn throttle(&self, opts: &BucketOptions) -> Option<BucketThrottle> {
+        self.t_lock
+            .read()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor rwlock read poisoned, recovering");
+                e.into_inner()
+            })
+            .get(opts)
+            .cloned()
+    }
+
+    pub fn set_bandwidth_limit(&self, bucket: &str, arn: &str, limit: i64) {
+        if limit <= 0 {
+            warn!(
+                bucket = bucket,
+                arn = arn,
+                limit = limit,
+                "invalid bandwidth limit, must be positive; ignoring"
+            );
+            return;
+        }
+        let limit_bytes = limit / self.node_count as i64;
+        if limit_bytes == 0 && limit > 0 {
+            warn!(
+                bucket = bucket,
+                arn = arn,
+                limit = limit,
+                node_count = self.node_count,
+                "bandwidth limit too small for cluster size, per-node limit will clamp to 1 byte/s"
+            );
+        }
+        let opts = BucketOptions {
+            name: bucket.to_string(),
+            replication_arn: arn.to_string(),
+        };
+        let throttle = match BucketThrottle::new(limit_bytes) {
+            Ok(t) => t,
+            Err(e) => {
+                warn!(
+                    bucket = bucket,
+                    arn = arn,
+                    limit_bytes = limit_bytes,
+                    err = %e,
+                    "failed to build bandwidth throttle, throttling disabled for this target"
+                );
+                return;
+            }
+        };
+        self.t_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .insert(opts, throttle);
+    }
+
+    pub fn is_throttled(&self, bucket: &str, arn: &str) -> bool {
+        let opt = BucketOptions {
+            name: bucket.to_string(),
+            replication_arn: arn.to_string(),
+        };
+        self.t_lock
+            .read()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor rwlock read poisoned, recovering");
+                e.into_inner()
+            })
+            .contains_key(&opt)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_set_and_get_throttle_with_node_split() {
+        let monitor = Monitor::new(4);
+        monitor.set_bandwidth_limit("b1", "arn1", 400);
+
+        let throttle = monitor
+            .throttle(&BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            })
+            .expect("throttle should exist");
+
+        assert_eq!(throttle.node_bandwidth_per_sec, 100);
+        assert!(monitor.is_throttled("b1", "arn1"));
+    }
+
+    #[test]
+    fn test_delete_bucket_throttle() {
+        let monitor = Monitor::new(2);
+        monitor.set_bandwidth_limit("b1", "arn1", 200);
+        assert!(monitor.is_throttled("b1", "arn1"));
+
+        monitor.delete_bucket_throttle("b1", "arn1");
+        assert!(!monitor.is_throttled("b1", "arn1"));
+    }
+
+    #[test]
+    fn test_delete_bucket_removes_all_arns() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 100);
+        monitor.set_bandwidth_limit("b1", "arn2", 100);
+        monitor.set_bandwidth_limit("b2", "arn3", 100);
+
+        monitor.delete_bucket("b1");
+
+        assert!(!monitor.is_throttled("b1", "arn1"));
+        assert!(!monitor.is_throttled("b1", "arn2"));
+        assert!(monitor.is_throttled("b2", "arn3"));
+    }
+
+    #[test]
+    fn test_set_bandwidth_limit_ignores_non_positive() {
+        let monitor = Monitor::new(2);
+
+        monitor.set_bandwidth_limit("b1", "arn1", 0);
+        assert!(!monitor.is_throttled("b1", "arn1"));
+
+        monitor.set_bandwidth_limit("b1", "arn1", -10);
+        assert!(!monitor.is_throttled("b1", "arn1"));
+    }
+
+    #[test]
+    fn test_consume_returns_deficit_when_tokens_exhausted() {
+        let throttle = BucketThrottle::new(100).expect("test");
+
+        let (deficit, rate, _consumed) = throttle.consume(200);
+
+        assert!(deficit > 0);
+        assert!(rate > 0.0);
+    }
+
+    #[test]
+    fn test_consume_no_deficit_when_tokens_sufficient() {
+        let throttle = BucketThrottle::new(10000).expect("test");
+
+        std::thread::sleep(std::time::Duration::from_millis(1100));
+
+        let (deficit, _rate, _consumed) = throttle.consume(5000);
+        assert_eq!(deficit, 0);
+    }
+
+    #[test]
+    fn test_burst_equals_bandwidth() {
+        let throttle = BucketThrottle::new(500).expect("test");
+        assert_eq!(throttle.burst(), 500);
+    }
+
+    #[test]
+    fn test_concurrent_consume() {
+        let throttle = BucketThrottle::new(10000).expect("test");
+        std::thread::sleep(std::time::Duration::from_millis(1100));
+
+        let mut handles = vec![];
+        for _ in 0..10 {
+            let t = throttle.clone();
+            handles.push(std::thread::spawn(move || t.consume(100)));
+        }
+
+        let mut total_deficit = 0u64;
+        let mut total_consumed = 0u64;
+        for h in handles {
+            let (deficit, _rate, consumed) = h.join().unwrap();
+            total_consumed += consumed;
+            total_deficit += deficit;
+        }
+
+        assert_eq!(total_consumed + total_deficit, 1000);
+        assert!(total_consumed <= 10000);
+    }
+
+    #[test]
+    fn test_zero_bandwidth_clamped_to_one() {
+        let throttle = BucketThrottle::new(0).expect("test");
+        assert_eq!(throttle.burst(), 1);
+        assert_eq!(throttle.node_bandwidth_per_sec, 1);
+    }
+
+    #[test]
+    fn test_negative_bandwidth_clamped_to_one() {
+        let throttle = BucketThrottle::new(-100).expect("test");
+        assert_eq!(throttle.burst(), 1);
+        assert_eq!(throttle.node_bandwidth_per_sec, 1);
+    }
+
+    #[test]
+    fn test_update_bandwidth_limit_overrides_previous() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 1000);
+
+        let opts = BucketOptions {
+            name: "b1".to_string(),
+            replication_arn: "arn1".to_string(),
+        };
+        let t1 = monitor.throttle(&opts).expect("throttle should exist");
+        assert_eq!(t1.burst(), 1000);
+        assert_eq!(t1.node_bandwidth_per_sec, 1000);
+
+        monitor.set_bandwidth_limit("b1", "arn1", 500);
+
+        let t2 = monitor.throttle(&opts).expect("throttle should exist after update");
+        assert_eq!(t2.burst(), 500);
+        assert_eq!(t2.node_bandwidth_per_sec, 500);
+    }
+}
@@ -0,0 +1,336 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::bucket::bandwidth::monitor::Monitor;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+use tokio::io::{AsyncRead, ReadBuf};
+use tokio::time::Sleep;
+use tracing::{debug, warn};
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct BucketOptions {
+    pub name: String,
+    pub replication_arn: String,
+}
+
+pub struct MonitorReaderOptions {
+    pub bucket_options: BucketOptions,
+    pub header_size: usize,
+}
+
+struct WaitState {
+    sleep: Pin<Box<Sleep>>,
+}
+
+pub struct MonitoredReader<R> {
+    r: R,
+    m: Arc<Monitor>,
+    opts: MonitorReaderOptions,
+    wait_state: std::sync::Mutex<Option<WaitState>>,
+    temp_buf: Vec<u8>,
+}
+
+impl<R> MonitoredReader<R> {
+    pub fn new(m: Arc<Monitor>, r: R, opts: MonitorReaderOptions) -> Self {
+        let throttle = m.throttle(&opts.bucket_options);
+        debug!(
+            bucket = opts.bucket_options.name,
+            arn = opts.bucket_options.replication_arn,
+            throttle_active = throttle.is_some(),
+            limit_bps = throttle.as_ref().map(|t| t.node_bandwidth_per_sec).unwrap_or(0),
+            header_size = opts.header_size,
+            "MonitoredReader created"
+        );
+        MonitoredReader {
+            r,
+            m,
+            opts,
+            wait_state: std::sync::Mutex::new(None),
+            temp_buf: Vec::new(),
+        }
+    }
+}
+
+impl<R: AsyncRead + Unpin> AsyncRead for MonitoredReader<R> {
+    fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+        let this = self.get_mut();
+
+        {
+            let mut guard = this.wait_state.lock().unwrap_or_else(|e| {
+                warn!("MonitoredReader wait_state mutex poisoned, recovering");
+                e.into_inner()
+            });
+            if let Some(ref mut ws) = *guard {
+                match ws.sleep.as_mut().poll(cx) {
+                    Poll::Pending => return Poll::Pending,
+                    Poll::Ready(()) => {
+                        *guard = None;
+                        drop(guard);
+                    }
+                }
+            }
+        }
+
+        let throttle = match this.m.throttle(&this.opts.bucket_options) {
+            Some(t) => t,
+            None => return Pin::new(&mut this.r).poll_read(cx, buf),
+        };
+
+        let b = throttle.burst();
+        debug_assert!(b >= 1, "burst must be at least 1");
+        let (need, tokens) = calc_need_and_tokens(b, buf.remaining(), &mut this.opts.header_size);
+        let (deficit, rate, consumed) = throttle.consume(tokens);
+        let need = need.min(consumed as usize);
+
+        if deficit > 0 && rate > 0.0 {
+            let duration = std::time::Duration::from_secs_f64(deficit as f64 / rate);
+            debug!(
+                tokens = tokens,
+                deficit = deficit,
+                rate = rate,
+                sleep_ms = duration.as_millis() as u64,
+                "bandwidth throttle sleep"
+            );
+            let mut sleep = Box::pin(tokio::time::sleep(duration));
+            match sleep.as_mut().poll(cx) {
+                Poll::Pending => {
+                    *this.wait_state.lock().unwrap_or_else(|e| {
+                        warn!("MonitoredReader wait_state mutex poisoned, recovering");
+                        e.into_inner()
+                    }) = Some(WaitState { sleep });
+                    return Poll::Pending;
+                }
+                Poll::Ready(()) => {}
+            }
+        }
+
+        poll_limited_read(&mut this.r, cx, buf, need, &mut this.temp_buf)
+    }
+}
+
+fn calc_need_and_tokens(burst: u64, need_upper: usize, header_size: &mut usize) -> (usize, u64) {
+    let hdr = *header_size;
+    let mut need = need_upper;
+    let tokens: u64 = if hdr > 0 {
+        if (hdr as u64) < burst {
+            *header_size = 0;
+            need = ((burst - hdr as u64) as usize).min(need);
+            need as u64 + hdr as u64
+        } else {
+            *header_size -= burst as usize;
+            need = 0;
+            burst
+        }
+    } else {
+        need = need.min(burst as usize);
+        need as u64
+    };
+    (need, tokens)
+}
+
+fn poll_limited_read<R: AsyncRead + Unpin>(
+    r: &mut R,
+    cx: &mut Context<'_>,
+    buf: &mut ReadBuf<'_>,
+    limit: usize,
+    reusable_buf: &mut Vec<u8>,
+) -> Poll<std::io::Result<()>> {
+    let remaining = buf.remaining();
+    if limit == 0 || remaining == 0 {
+        return Poll::Ready(Ok(()));
+    }
+
+    if remaining <= limit {
+        return Pin::new(r).poll_read(cx, buf);
+    }
+
+    reusable_buf.resize(limit, 0);
+    let mut temp_buf = ReadBuf::new(&mut reusable_buf[..limit]);
+    match Pin::new(r).poll_read(cx, &mut temp_buf) {
+        Poll::Ready(Ok(())) => {
+            buf.put_slice(temp_buf.filled());
+            Poll::Ready(Ok(()))
+        }
+        other => other,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures_util::task::noop_waker_ref;
+    use std::io;
+    use std::pin::Pin;
+    use std::task::Context;
+    use tokio::io::AsyncReadExt;
+
+    #[derive(Default)]
+    struct TestAsyncReader {
+        data: Vec<u8>,
+        pos: usize,
+    }
+
+    impl TestAsyncReader {
+        fn new(data: &[u8]) -> Self {
+            Self {
+                data: data.to_vec(),
+                pos: 0,
+            }
+        }
+    }
+
+    impl AsyncRead for TestAsyncReader {
+        fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
+            if self.pos >= self.data.len() {
+                return Poll::Ready(Ok(()));
+            }
+
+            let remaining = self.data.len() - self.pos;
+            let n = remaining.min(buf.remaining());
+            buf.put_slice(&self.data[self.pos..self.pos + n]);
+            self.pos += n;
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_passthrough_when_throttle_absent() {
+        let monitor = Monitor::new(1);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 0,
+        };
+        let inner = TestAsyncReader::new(b"hello-world");
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out, b"hello-world");
+    }
+
+    #[test]
+    fn test_poll_limited_read_honors_limit() {
+        let mut inner = TestAsyncReader::new(b"abcdef");
+        let mut out = [0u8; 8];
+        let mut reusable = Vec::new();
+        let waker = noop_waker_ref();
+        let mut cx = Context::from_waker(waker);
+
+        let mut read_buf = ReadBuf::new(&mut out);
+        let first = poll_limited_read(&mut inner, &mut cx, &mut read_buf, 3, &mut reusable);
+        assert!(matches!(first, Poll::Ready(Ok(()))));
+        assert_eq!(read_buf.filled(), b"abc");
+
+        let second = poll_limited_read(&mut inner, &mut cx, &mut read_buf, 3, &mut reusable);
+        assert!(matches!(second, Poll::Ready(Ok(()))));
+        assert_eq!(read_buf.filled(), b"abcdef");
+    }
+
+    #[tokio::test]
+    async fn test_header_only_consumption_skips_io() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 10);
+
+        let data = vec![0xAAu8; 20];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 25,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let waker = noop_waker_ref();
+        let mut cx = Context::from_waker(waker);
+        let mut out = [0u8; 16];
+        let mut read_buf = ReadBuf::new(&mut out);
+
+        let poll = Pin::new(&mut reader).poll_read(&mut cx, &mut read_buf);
+        assert!(matches!(poll, Poll::Ready(Ok(())) | Poll::Pending));
+        assert!(read_buf.filled().is_empty());
+        assert_eq!(reader.opts.header_size, 15);
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_with_throttle_reads_all_data() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 1024);
+
+        let data = vec![0xABu8; 4096];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 0,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out.len(), 4096);
+        assert!(out.iter().all(|&b| b == 0xAB));
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_header_size_accounting() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 100);
+
+        let data = vec![0u8; 200];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 50,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out.len(), 200);
+        assert_eq!(reader.opts.header_size, 0);
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_very_small_limit() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 1);
+
+        let data = vec![0xFFu8; 10];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 0,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out.len(), 10);
+    }
+}
@@ -21,6 +21,7 @@ use crate::bucket::target::ARN;
 use crate::bucket::target::BucketTargetType;
 use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials};
 use crate::bucket::versioning_sys::BucketVersioningSys;
+use crate::global::get_global_bucket_monitor;
 use aws_credential_types::Credentials as SdkCredentials;
 use aws_sdk_s3::config::Region as SdkRegion;
 use aws_sdk_s3::error::ProvideErrorMetadata;
@@ -35,8 +36,10 @@ use aws_sdk_s3::types::{
 };
 use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput};
 use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus};
+use aws_smithy_http_client::{Builder as SmithyHttpClientBuilder, tls as smithy_tls};
 use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
 use reqwest::Client as HttpClient;
+use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
 use rustfs_filemeta::{ReplicationStatusType, ReplicationType};
 use rustfs_utils::http::{
    AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
@@ -49,6 +52,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::error::Error;
 use std::fmt;
+use std::path::Path;
 use std::str::FromStr as _;
 use std::sync::Arc;
 use std::sync::OnceLock;
@@ -64,6 +68,8 @@ use uuid::Uuid;
 const DEFAULT_HEALTH_CHECK_DURATION: Duration = Duration::from_secs(5);
 const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);

+const REPLICATION_REQUEST_TRUE: HeaderValue = HeaderValue::from_static("true");
+
 pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();

 #[derive(Debug, Clone)]
@@ -637,12 +643,23 @@ impl BucketTargetSys {
            format!("http://{}", target.endpoint)
        };

-        let config = S3Config::builder()
+        let mut config_builder = S3Config::builder()
            .endpoint_url(endpoint.clone())
            .credentials_provider(SharedCredentialsProvider::new(creds))
            .region(SdkRegion::new(target.region.clone()))
-            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
-            .build();
+            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
+
+        if should_force_path_style(target) {
+            config_builder = config_builder.force_path_style(true);
+        }
+
+        if target.secure
+            && let Some(http_client) = build_aws_s3_http_client_from_tls_path().await
+        {
+            config_builder = config_builder.http_client(http_client);
+        }
+
+        let config = config_builder.build();

        Ok(TargetClient {
            endpoint,
@@ -665,9 +682,19 @@ impl BucketTargetSys {
        Ok(true)
    }

-    fn update_bandwidth_limit(&self, _bucket: &str, _arn: &str, _limit: i64) {
-        // Implementation for bandwidth limit update
-        // This would interact with the global bucket monitor
+    fn update_bandwidth_limit(&self, bucket: &str, arn: &str, limit: i64) {
+        if let Some(bucket_monitor) = get_global_bucket_monitor() {
+            if limit == 0 {
+                bucket_monitor.delete_bucket_throttle(bucket, arn);
+                return;
+            }
+            bucket_monitor.set_bandwidth_limit(bucket, arn, limit);
+        } else {
+            error!(
+                "Global bucket monitor uninitialized; skipping bandwidth limit update for bucket '{}' and ARN '{}'",
+                bucket, arn
+            );
+        }
    }

    pub async fn get_remote_target_client_by_arn(&self, _bucket: &str, arn: &str) -> Option<Arc<TargetClient>> {
@@ -689,6 +716,7 @@ impl BucketTargetSys {
        if let Some(existing_targets) = targets_map.remove(bucket) {
            for target in existing_targets {
                arn_remotes_map.remove(&target.arn);
+                self.update_bandwidth_limit(bucket, &target.arn, 0);
            }
        }

@@ -775,6 +803,72 @@ impl BucketTargetSys {
    }
 }

+async fn build_aws_s3_http_client_from_tls_path() -> Option<aws_sdk_s3::config::SharedHttpClient> {
+    let tls_path = std::env::var("RUSTFS_TLS_PATH").ok()?;
+    if tls_path.is_empty() {
+        return None;
+    }
+
+    let tls_dir = Path::new(&tls_path);
+    let mut trust_store = smithy_tls::TrustStore::default();
+    let mut has_custom_certs = false;
+
+    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
+    match tokio::fs::read(&ca_path).await {
+        Ok(pem) => {
+            trust_store.add_pem_certificate(pem);
+            has_custom_certs = true;
+        }
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
+    }
+
+    if rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) {
+        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
+        match tokio::fs::read(&leaf_cert_path).await {
+            Ok(pem) => {
+                trust_store.add_pem_certificate(pem);
+                has_custom_certs = true;
+            }
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
+        }
+    }
+
+    if !has_custom_certs {
+        return None;
+    }
+
+    let tls_context = match smithy_tls::TlsContext::builder().with_trust_store(trust_store).build() {
+        Ok(ctx) => ctx,
+        Err(e) => {
+            warn!("failed to build AWS SDK TLS context for replication client: {}", e);
+            return None;
+        }
+    };
+
+    Some(
+        SmithyHttpClientBuilder::new()
+            .tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
+            .tls_context(tls_context)
+            .build_https(),
+    )
+}
+
+fn should_force_path_style(target: &BucketTarget) -> bool {
+    match target.path.trim().to_ascii_lowercase().as_str() {
+        // Explicit DNS/virtual-hosted-style requested by user.
+        "dns" | "off" | "false" => false,
+        // Explicit path-style or legacy boolean-like values.
+        "path" | "on" | "true" => true,
+        // `auto` and empty are defaulted to path-style for custom S3-compatible endpoints.
+        // RustFS/MinIO-style deployments typically do not configure virtual-hosted-style routing.
+        "auto" | "" => true,
+        // Unknown values: prefer compatibility with S3-compatible services.
+        _ => true,
+    }
+}
+
 // generate ARN that is unique to this target type
 fn generate_arn(t: &BucketTarget, depl_id: &str) -> String {
    let uuid = if depl_id.is_empty() {
@@ -999,10 +1093,14 @@ impl PutObjectOptions {
        if self.internal.source_mtime.unix_timestamp() != 0 {
            header.insert(
                RUSTFS_BUCKET_SOURCE_MTIME,
-                HeaderValue::from_str(&self.internal.source_mtime.unix_timestamp().to_string()).expect("err"),
+                HeaderValue::from_str(&self.internal.source_mtime.format(&Rfc3339).unwrap_or_default()).expect("err"),
            );
        }

+        if self.internal.replication_request {
+            header.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE);
+        }
+
        header
    }

@@ -1211,6 +1309,9 @@ impl TargetClient {
        {
            headers.insert(RUSTFS_BUCKET_SOURCE_VERSION_ID, header_value);
        }
+        if opts.internal.replication_request {
+            headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE);
+        }

        match self
            .client
@@ -34,7 +34,7 @@ use crate::global::GLOBAL_LocalNodeName;
 use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
 use crate::store::ECStore;
 use crate::store_api::StorageAPI;
-use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
+use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete};
 use crate::tier::warm_backend::WarmBackendGetOpts;
 use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
 use bytes::BytesMut;
@@ -65,7 +65,7 @@ use time::OffsetDateTime;
 use tokio::select;
 use tokio::sync::mpsc::{Receiver, Sender};
 use tokio::sync::{RwLock, mpsc};
-use tracing::{debug, error, info};
+use tracing::{debug, error, info, warn};
 use uuid::Uuid;
 use xxhash_rust::xxh64;

@@ -102,7 +102,7 @@ impl LifecycleSys {
    }

    pub fn trace(_oi: &ObjectInfo) -> TraceFn {
-        todo!();
+        Arc::new(|_oi, _ctx| Box::pin(async move {}))
    }
 }

@@ -395,7 +395,7 @@ impl ExpiryState {
                    }
                    else {
                        //info!("Invalid work type - {:?}", v);
-                        todo!();
+                        warn!("lifecycle worker received unsupported operation type");
                    }
                }
            }
@@ -782,12 +782,13 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
        mod_time: oi.mod_time,
        ..Default::default()
    };
-    time_ilm(1);
-    api.transition_object(&oi.bucket, &oi.name, &opts).await
+    let result = api.transition_object(&oi.bucket, &oi.name, &opts).await;
+    time_ilm(1)();
+    result
 }

 pub fn audit_tier_actions(_api: ECStore, _tier: &str, _bytes: i64) -> TimeFn {
-    todo!();
+    Arc::new(|| Box::pin(async move {}))
 }

 pub async fn get_transitioned_object_reader(
@@ -1082,11 +1083,11 @@ pub async fn apply_expiry_on_transitioned_object(
    lc_event: &lifecycle::Event,
    src: &LcEventSrc,
 ) -> bool {
-    // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
+    let time_ilm = Metrics::time_ilm(lc_event.action);
    if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await {
        return false;
    }
-    // let _ = time_ilm(1);
+    time_ilm(1)();

    true
 }
@@ -1114,7 +1115,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
        opts.delete_prefix_object = true;
    }

-    // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
+    let time_ilm = Metrics::time_ilm(lc_event.action);

    //debug!("lc_event.action: {:?}", lc_event.action);
    //debug!("opts: {:?}", opts);
@@ -1152,11 +1153,11 @@ pub async fn apply_expiry_on_non_transitioned_objects(
    });

    if lc_event.action != lifecycle::IlmAction::NoneAction {
-        // let mut num_versions = 1_u64;
-        // if lc_event.action.delete_all() {
-        //     num_versions = oi.num_versions as u64;
-        // }
-        // let _ = time_ilm(num_versions);
+        let mut num_versions = 1_u64;
+        if lc_event.action.delete_all() {
+            num_versions = oi.num_versions as u64;
+        }
+        time_ilm(num_versions)();
    }

    true
@@ -18,12 +18,10 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use crate::bucket::lifecycle::rule::TransitionOps;
-use crate::store_api::ObjectInfo;
 use rustfs_filemeta::{ReplicationStatusType, VersionPurgeStatusType};
 use s3s::dto::{
    BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
-    ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition,
+    ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition, TransitionStorageClass,
 };
 use std::cmp::Ordering;
 use std::collections::HashMap;
@@ -35,6 +33,9 @@ use time::{self, Duration, OffsetDateTime};
 use tracing::info;
 use uuid::Uuid;

+use crate::bucket::lifecycle::rule::TransitionOps;
+use crate::store_api::ObjectInfo;
+
 pub const TRANSITION_COMPLETE: &str = "complete";
 pub const TRANSITION_PENDING: &str = "pending";
 const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
@@ -44,6 +45,10 @@ const _ERR_XML_NOT_WELL_FORMED: &str =
 const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
    "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an retention bucket";
 const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
+const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days must be greater than 0";
+const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
+const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters";
+const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";

 pub use rustfs_common::metrics::IlmAction;

@@ -137,6 +142,7 @@ pub trait Lifecycle {
    async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error>;
    async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>>;
    async fn eval(&self, obj: &ObjectOpts) -> Event;
+    async fn predict_expiration(&self, obj: &ObjectOpts) -> Event;
    async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime, newer_noncurrent_versions: usize) -> Event;
    //fn set_prediction_headers(&self, w: http.ResponseWriter, obj: ObjectOpts);
    async fn noncurrent_versions_expiration_limit(self: Arc<Self>, obj: &ObjectOpts) -> Event;
@@ -171,44 +177,51 @@ impl Lifecycle for BucketLifecycleConfiguration {
                continue;
            }

-            let rule_prefix = rule.prefix.as_ref().expect("err!");
+            let rule_prefix = &rule.prefix.clone().unwrap_or_default();
            if prefix.len() > 0 && rule_prefix.len() > 0 && !prefix.starts_with(rule_prefix) && !rule_prefix.starts_with(&prefix)
            {
                continue;
            }

-            let rule_noncurrent_version_expiration = rule.noncurrent_version_expiration.as_ref().expect("err!");
-            if rule_noncurrent_version_expiration.noncurrent_days.expect("err!") > 0 {
+            if let Some(rule_noncurrent_version_expiration) = &rule.noncurrent_version_expiration {
+                if let Some(noncurrent_days) = rule_noncurrent_version_expiration.noncurrent_days {
+                    if noncurrent_days > 0 {
+                        return true;
+                    }
+                }
+                if let Some(newer_noncurrent_versions) = rule_noncurrent_version_expiration.newer_noncurrent_versions {
+                    if newer_noncurrent_versions > 0 {
+                        return true;
+                    }
+                }
+            }
+            if rule.noncurrent_version_transitions.is_some() {
                return true;
            }
-            if rule_noncurrent_version_expiration.newer_noncurrent_versions.expect("err!") > 0 {
-                return true;
+            if let Some(rule_expiration) = &rule.expiration {
+                if let Some(date1) = rule_expiration.date.clone() {
+                    if OffsetDateTime::from(date1).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() {
+                        return true;
+                    }
+                }
+                if rule_expiration.date.is_some() {
+                    return true;
+                }
+                if let Some(expired_object_delete_marker) = rule_expiration.expired_object_delete_marker
+                    && expired_object_delete_marker
+                {
+                    return true;
+                }
            }
-            if !rule.noncurrent_version_transitions.is_none() {
-                return true;
+            if let Some(rule_transitions) = &rule.transitions {
+                let rule_transitions_0 = rule_transitions[0].clone();
+                if let Some(date1) = rule_transitions_0.date {
+                    if OffsetDateTime::from(date1).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() {
+                        return true;
+                    }
+                }
            }
-            let rule_expiration = rule.expiration.as_ref().expect("err!");
-            if !rule_expiration.date.is_none()
-                && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp()
-                    < OffsetDateTime::now_utc().unix_timestamp()
-            {
-                return true;
-            }
-            if !rule_expiration.date.is_none() {
-                return true;
-            }
-            if rule_expiration.expired_object_delete_marker.expect("err!") {
-                return true;
-            }
-            let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!");
-            let rule_transitions_0 = rule_transitions[0].clone();
-            if !rule_transitions_0.date.is_none()
-                && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp()
-                    < OffsetDateTime::now_utc().unix_timestamp()
-            {
-                return true;
-            }
-            if !rule.transitions.is_none() {
+            if rule.transitions.is_some() {
                return true;
            }
        }
@@ -224,6 +237,30 @@ impl Lifecycle for BucketLifecycleConfiguration {
        }

        for r in &self.rules {
+            if r.status != ExpirationStatus::from_static(ExpirationStatus::ENABLED)
+                && r.status != ExpirationStatus::from_static(ExpirationStatus::DISABLED)
+            {
+                return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_STATUS));
+            }
+
+            if let Some(expiration) = &r.expiration {
+                if let Some(expiration_date) = &expiration.date {
+                    let date = OffsetDateTime::from(expiration_date.clone());
+                    if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT));
+                    }
+                }
+                if let Some(days) = expiration.days {
+                    if days <= 0 {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS));
+                    }
+                }
+            }
+            if let Some(id) = &r.id {
+                if id.len() > 255 {
+                    return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG));
+                }
+            }
            r.validate()?;
            /*if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() {
                if let Some(expiration) = r.expiration.as_ref() {
@@ -232,7 +269,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                            return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
                        }
                    }
-                        }
+                }
            }*/
        }
        for (i, _) in self.rules.iter().enumerate() {
@@ -241,8 +278,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
            }
            let other_rules = &self.rules[i + 1..];
            for other_rule in other_rules {
-                if self.rules[i].id == other_rule.id {
-                    return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
+                if let (Some(id1), Some(id2)) = (&self.rules[i].id, &other_rule.id) {
+                    if id1 == id2 {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
+                    }
                }
            }
        }
@@ -279,6 +318,61 @@ impl Lifecycle for BucketLifecycleConfiguration {
        self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await
    }

+    async fn predict_expiration(&self, obj: &ObjectOpts) -> Event {
+        let mod_time = match obj.mod_time {
+            Some(time) => {
+                if time.unix_timestamp() == 0 {
+                    return Event::default();
+                }
+                time
+            }
+            None => return Event::default(),
+        };
+
+        if obj.delete_marker || !(obj.is_latest || obj.version_id.is_none_or(|v| v.is_nil())) {
+            return Event::default();
+        }
+
+        let Some(lc_rules) = self.filter_rules(obj).await else {
+            return Event::default();
+        };
+
+        let mut event: Option<Event> = None;
+        for rule in lc_rules {
+            let Some(expiration) = rule.expiration else {
+                continue;
+            };
+            if expiration.expired_object_delete_marker.is_some_and(|v| v) {
+                continue;
+            }
+            let Some(due) = (match (&expiration.date, expiration.days) {
+                (Some(date), _) => Some(OffsetDateTime::from(date.clone())),
+                (None, Some(days)) => Some(expected_expiry_time(mod_time, days)),
+                _ => None,
+            }) else {
+                continue;
+            };
+
+            let predicted = Event {
+                action: IlmAction::DeleteAction,
+                rule_id: rule.id.clone().unwrap_or_default(),
+                due: Some(due),
+                noncurrent_days: 0,
+                newer_noncurrent_versions: 0,
+                storage_class: "".into(),
+            };
+            let predicted_due = predicted.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp();
+            let should_replace = event
+                .as_ref()
+                .is_none_or(|e| predicted_due < e.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp());
+            if should_replace {
+                event = Some(predicted);
+            }
+        }
+
+        event.unwrap_or_default()
+    }
+
    async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime, newer_noncurrent_versions: usize) -> Event {
        let mut events = Vec::<Event>::new();
        info!(
@@ -325,7 +419,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                        if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
                            events.push(Event {
                                action: IlmAction::DeleteVersionAction,
-                                rule_id: rule.id.clone().expect("err!"),
+                                rule_id: rule.id.clone().unwrap_or_default(),
                                due: Some(now),
                                noncurrent_days: 0,
                                newer_noncurrent_versions: 0,
@@ -339,7 +433,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                            if now.unix_timestamp() >= expected_expiry.unix_timestamp() {
                                events.push(Event {
                                    action: IlmAction::DeleteVersionAction,
-                                    rule_id: rule.id.clone().expect("err!"),
+                                    rule_id: rule.id.clone().unwrap_or_default(),
                                    due: Some(expected_expiry),
                                    noncurrent_days: 0,
                                    newer_noncurrent_versions: 0,
@@ -360,7 +454,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                    if now.unix_timestamp() >= due.unix_timestamp() {
                                        events.push(Event {
                                            action: IlmAction::DelMarkerDeleteAllVersionsAction,
-                                            rule_id: rule.id.clone().expect("err!"),
+                                            rule_id: rule.id.clone().unwrap_or_default(),
                                            due: Some(due),
                                            noncurrent_days: 0,
                                            newer_noncurrent_versions: 0,
@@ -393,7 +487,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                    if now.unix_timestamp() >= expected_expiry.unix_timestamp() {
                                        events.push(Event {
                                            action: IlmAction::DeleteVersionAction,
-                                            rule_id: rule.id.clone().expect("err!"),
+                                            rule_id: rule.id.clone().unwrap_or_default(),
                                            due: Some(expected_expiry),
                                            noncurrent_days: 0,
                                            newer_noncurrent_versions: 0,
@@ -416,7 +510,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                    if now.unix_timestamp() == 0 || now.unix_timestamp() > due0.unix_timestamp() {
                                        events.push(Event {
                                            action: IlmAction::TransitionVersionAction,
-                                            rule_id: rule.id.clone().expect("err!"),
+                                            rule_id: rule.id.clone().unwrap_or_default(),
                                            due,
                                            storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0]
                                                .storage_class
@@ -450,7 +544,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                info!("eval_inner: expiration by date - date0={:?}", date0);
                                events.push(Event {
                                    action: IlmAction::DeleteAction,
-                                    rule_id: rule.id.clone().expect("err!"),
+                                    rule_id: rule.id.clone().unwrap_or_default(),
                                    due: Some(date0),
                                    noncurrent_days: 0,
                                    newer_noncurrent_versions: 0,
@@ -471,7 +565,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                info!("eval_inner: object should expire, adding DeleteAction");
                                let mut event = Event {
                                    action: IlmAction::DeleteAction,
-                                    rule_id: rule.id.clone().expect("err!"),
+                                    rule_id: rule.id.clone().unwrap_or_default(),
                                    due: Some(expected_expiry),
                                    noncurrent_days: 0,
                                    newer_noncurrent_versions: 0,
@@ -496,9 +590,14 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                if now.unix_timestamp() == 0 || now.unix_timestamp() > due0.unix_timestamp() {
                                    events.push(Event {
                                        action: IlmAction::TransitionAction,
-                                        rule_id: rule.id.clone().expect("err!"),
+                                        rule_id: rule.id.clone().unwrap_or_default(),
                                        due,
-                                        storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(),
+                                        storage_class: transitions[0]
+                                            .storage_class
+                                            .clone()
+                                            .unwrap_or(TransitionStorageClass::from_static(""))
+                                            .as_str()
+                                            .to_string(),
                                        noncurrent_days: 0,
                                        newer_noncurrent_versions: 0,
                                    });
@@ -512,9 +611,10 @@ impl Lifecycle for BucketLifecycleConfiguration {

        if events.len() > 0 {
            events.sort_by(|a, b| {
-                if now.unix_timestamp() > a.due.expect("err!").unix_timestamp()
-                    && now.unix_timestamp() > b.due.expect("err").unix_timestamp()
-                    || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp()
+                if now.unix_timestamp() > a.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
+                    && now.unix_timestamp() > b.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
+                    || a.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
+                        == b.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
                {
                    match a.action {
                        IlmAction::DeleteAllVersionsAction
@@ -537,7 +637,9 @@ impl Lifecycle for BucketLifecycleConfiguration {
                    return Ordering::Less;
                }

-                if a.due.expect("err").unix_timestamp() < b.due.expect("err").unix_timestamp() {
+                if a.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
+                    < b.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
+                {
                    return Ordering::Less;
                }
                return Ordering::Greater;
@@ -558,8 +660,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
                        }
                        return Event {
                            action: IlmAction::DeleteVersionAction,
-                            rule_id: rule.id.clone().expect("err"),
-                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
+                            rule_id: rule.id.clone().unwrap_or_default(),
+                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.unwrap_or(0) as u32,
                            newer_noncurrent_versions: newer_noncurrent_versions as usize,
                            due: Some(OffsetDateTime::UNIX_EPOCH),
                            storage_class: "".into(),
@@ -567,8 +669,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
                    } else {
                        return Event {
                            action: IlmAction::DeleteVersionAction,
-                            rule_id: rule.id.clone().expect("err"),
-                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
+                            rule_id: rule.id.clone().unwrap_or_default(),
+                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.unwrap_or(0) as u32,
                            newer_noncurrent_versions: 0,
                            due: Some(OffsetDateTime::UNIX_EPOCH),
                            storage_class: "".into(),
@@ -754,3 +856,222 @@ impl Default for TransitionOptions {
        }
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn validate_rejects_non_positive_expiration_days() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(0),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc
+            .validate(&ObjectLockConfiguration::default())
+            .await
+            .expect_err("expected validation error");
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS);
+    }
+
+    #[tokio::test]
+    async fn validate_accepts_positive_expiration_days() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        lc.validate(&ObjectLockConfiguration::default())
+            .await
+            .expect("expected validation to pass");
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_non_midnight_expiration_date() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    date: Some(time::OffsetDateTime::from_unix_timestamp(20_000_101).unwrap().into()),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT);
+    }
+
+    #[tokio::test]
+    async fn predict_expiration_selects_closest_expiry_for_put_object() {
+        let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        days: Some(30),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: Some("rule-days".to_string()),
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        date: Some((base_time + Duration::days(1)).into()),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: Some("rule-date".to_string()),
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+            ],
+        };
+        let opts = ObjectOpts {
+            name: "obj".to_string(),
+            mod_time: Some(base_time),
+            is_latest: true,
+            version_id: None,
+            ..Default::default()
+        };
+
+        let event = lc.predict_expiration(&opts).await;
+        let expected = base_time + Duration::days(1);
+
+        assert_eq!(event.action, IlmAction::DeleteAction);
+        assert_eq!(event.rule_id, "rule-date");
+        assert_eq!(event.due, Some(expected));
+    }
+
+    #[tokio::test]
+    async fn validate_accepts_multiple_rules_without_ids() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        days: Some(30),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: None,
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        days: Some(31),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: None,
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+            ],
+        };
+
+        lc.validate(&ObjectLockConfiguration::default())
+            .await
+            .expect("expected validation to pass");
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_rule_id_too_long() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: Some("a".repeat(256)),
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG);
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_invalid_status_case_sensitive() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static("enabled"),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_RULE_STATUS);
+    }
+}
@@ -26,7 +26,8 @@ use rmp_serde::Serializer as rmpSerializer;
 use rustfs_policy::policy::BucketPolicy;
 use s3s::dto::{
    BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration,
-    ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
+    PublicAccessBlockConfiguration, ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging,
+    VersioningConfiguration,
 };
 use serde::Serializer;
 use serde::{Deserialize, Serialize};
@@ -50,6 +51,8 @@ pub const BUCKET_VERSIONING_CONFIG: &str = "versioning.xml";
 pub const BUCKET_REPLICATION_CONFIG: &str = "replication.xml";
 pub const BUCKET_TARGETS_FILE: &str = "bucket-targets.json";
 pub const BUCKET_CORS_CONFIG: &str = "cors.xml";
+pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
+pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";

 #[derive(Debug, Deserialize, Serialize, Clone)]
 #[serde(rename_all = "PascalCase", default)]
@@ -69,6 +72,8 @@ pub struct BucketMetadata {
    pub bucket_targets_config_json: Vec<u8>,
    pub bucket_targets_config_meta_json: Vec<u8>,
    pub cors_config_xml: Vec<u8>,
+    pub public_access_block_config_xml: Vec<u8>,
+    pub bucket_acl_config_json: Vec<u8>,

    pub policy_config_updated_at: OffsetDateTime,
    pub object_lock_config_updated_at: OffsetDateTime,
@@ -82,6 +87,8 @@ pub struct BucketMetadata {
    pub bucket_targets_config_updated_at: OffsetDateTime,
    pub bucket_targets_config_meta_updated_at: OffsetDateTime,
    pub cors_config_updated_at: OffsetDateTime,
+    pub public_access_block_config_updated_at: OffsetDateTime,
+    pub bucket_acl_config_updated_at: OffsetDateTime,

    #[serde(skip)]
    pub new_field_updated_at: OffsetDateTime,
@@ -110,6 +117,10 @@ pub struct BucketMetadata {
    pub bucket_target_config_meta: Option<HashMap<String, String>>,
    #[serde(skip)]
    pub cors_config: Option<CORSConfiguration>,
+    #[serde(skip)]
+    pub public_access_block_config: Option<PublicAccessBlockConfiguration>,
+    #[serde(skip)]
+    pub bucket_acl_config: Option<String>,
 }

 impl Default for BucketMetadata {
@@ -130,6 +141,8 @@ impl Default for BucketMetadata {
            bucket_targets_config_json: Default::default(),
            bucket_targets_config_meta_json: Default::default(),
            cors_config_xml: Default::default(),
+            public_access_block_config_xml: Default::default(),
+            bucket_acl_config_json: Default::default(),
            policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
            object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
            encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
@@ -142,6 +155,8 @@ impl Default for BucketMetadata {
            bucket_targets_config_updated_at: OffsetDateTime::UNIX_EPOCH,
            bucket_targets_config_meta_updated_at: OffsetDateTime::UNIX_EPOCH,
            cors_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH,
            new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
            policy_config: Default::default(),
            notification_config: Default::default(),
@@ -155,6 +170,8 @@ impl Default for BucketMetadata {
            bucket_target_config: Default::default(),
            bucket_target_config_meta: Default::default(),
            cors_config: Default::default(),
+            public_access_block_config: Default::default(),
+            bucket_acl_config: Default::default(),
        }
    }
 }
@@ -254,6 +271,12 @@ impl BucketMetadata {
        if self.bucket_targets_config_meta_updated_at == OffsetDateTime::UNIX_EPOCH {
            self.bucket_targets_config_meta_updated_at = self.created
        }
+        if self.public_access_block_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.public_access_block_config_updated_at = self.created
+        }
+        if self.bucket_acl_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.bucket_acl_config_updated_at = self.created
+        }
    }

    pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
@@ -307,6 +330,14 @@ impl BucketMetadata {
                self.cors_config_xml = data;
                self.cors_config_updated_at = updated;
            }
+            BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => {
+                self.public_access_block_config_xml = data;
+                self.public_access_block_config_updated_at = updated;
+            }
+            BUCKET_ACL_CONFIG => {
+                self.bucket_acl_config_json = data;
+                self.bucket_acl_config_updated_at = updated;
+            }
            _ => return Err(Error::other(format!("config file not found : {config_file}"))),
        }

@@ -380,6 +411,15 @@ impl BucketMetadata {
        if !self.cors_config_xml.is_empty() {
            self.cors_config = Some(deserialize::<CORSConfiguration>(&self.cors_config_xml)?);
        }
+        if !self.public_access_block_config_xml.is_empty() {
+            self.public_access_block_config =
+                Some(deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)?);
+        }
+        if !self.bucket_acl_config_json.is_empty() {
+            let acl = String::from_utf8(self.bucket_acl_config_json.clone())
+                .map_err(|e| Error::other(format!("invalid UTF-8 in bucket ACL: {}", e)))?;
+            self.bucket_acl_config = Some(acl);
+        }

        Ok(())
    }
@@ -530,6 +570,15 @@ mod test {
        bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec();
        bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc();

+        // Add public access block configuration
+        let public_access_block_xml = r#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#;
+        bm.public_access_block_config_xml = public_access_block_xml.as_bytes().to_vec();
+        bm.public_access_block_config_updated_at = OffsetDateTime::now_utc();
+
+        let bucket_acl = r#"{"owner":{"id":"rustfsadmin","display_name":"RustFS Tester"},"grants":[{"grantee":{"grantee_type":"CanonicalUser","id":"rustfsadmin","display_name":"RustFS Tester","uri":null,"email_address":null},"permission":"FULL_CONTROL"}]}"#;
+        bm.bucket_acl_config_json = bucket_acl.as_bytes().to_vec();
+        bm.bucket_acl_config_updated_at = OffsetDateTime::now_utc();
+
        // Test serialization
        let buf = bm.marshal_msg().unwrap();
        assert!(!buf.is_empty(), "Serialized buffer should not be empty");
@@ -549,6 +598,8 @@ mod test {
        assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml);
        assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml);
        assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json);
+        assert_eq!(bm.public_access_block_config_xml, deserialized_bm.public_access_block_config_xml);
+        assert_eq!(bm.bucket_acl_config_json, deserialized_bm.bucket_acl_config_json);
        assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml);
        assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml);
        assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml);
@@ -15,13 +15,13 @@
 use super::metadata::{BucketMetadata, load_bucket_metadata};
 use super::quota::BucketQuota;
 use super::target::BucketTargets;
-use crate::StorageAPI as _;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_parse};
 use crate::bucket::utils::{deserialize, is_meta_bucketname};
 use crate::error::{Error, Result, is_err_bucket_not_found};
 use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_layer_fn};
 use crate::store::ECStore;
+use crate::store_api::HealOperations as _;
 use futures::future::join_all;
 use lazy_static::lazy_static;
 use rustfs_common::heal_channel::HealOpts;
@@ -29,7 +29,7 @@ use rustfs_policy::policy::BucketPolicy;
 use s3s::dto::ReplicationConfiguration;
 use s3s::dto::{
    BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration,
-    ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
+    PublicAccessBlockConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
 };
 use std::collections::HashSet;
 use std::sync::OnceLock;
@@ -53,6 +53,10 @@ pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
    GLOBAL_BucketMetadataSys.set(sys).unwrap();
 }

+pub fn get_global_bucket_metadata_sys() -> Option<Arc<RwLock<BucketMetadataSys>>> {
+    GLOBAL_BucketMetadataSys.get().cloned()
+}
+
 // panic if not init
 pub(super) fn get_bucket_metadata_sys() -> Result<Arc<RwLock<BucketMetadataSys>>> {
    if let Some(sys) = GLOBAL_BucketMetadataSys.get() {
@@ -105,6 +109,13 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
    bucket_meta_sys.get_bucket_policy_raw(bucket).await
 }

+pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_bucket_acl_config(bucket).await
+}
+
 pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -133,6 +144,13 @@ pub async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime
    bucket_meta_sys.get_tagging_config(bucket).await
 }

+pub async fn get_public_access_block_config(bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_public_access_block_config(bucket).await
+}
+
 pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -478,6 +496,16 @@ impl BucketMetadataSys {
        }
    }

+    pub async fn get_bucket_acl_config(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.bucket_acl_config {
+            Ok((config.clone(), bm.bucket_acl_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
    pub async fn get_tagging_config(&self, bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
        let (bm, _) = self.get_config(bucket).await?;

@@ -488,6 +516,16 @@ impl BucketMetadataSys {
        }
    }

+    pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.public_access_block_config {
+            Ok((config.clone(), bm.public_access_block_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
    pub async fn get_object_lock_config(&self, bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> {
        let (bm, _) = self.get_config(bucket).await?;

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+pub mod bandwidth;
 pub mod bucket_target_sys;
 pub mod error;
 pub mod lifecycle;
@@ -106,7 +106,7 @@ pub fn check_retention_for_modification(
    None
 }

-pub(crate) fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
+pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
    let target_year = dt.year() + years;
    dt.replace_year(target_year)
        .or_else(|_| {
@@ -36,6 +36,18 @@ impl QuotaChecker {
        bucket: &str,
        operation: QuotaOperation,
        operation_size: u64,
+    ) -> Result<QuotaCheckResult, QuotaError> {
+        self.check_quota_with_usage_reporting(bucket, operation, operation_size, false)
+            .await
+    }
+
+    /// Check quota with option to force usage calculation even when no quota is configured
+    pub async fn check_quota_with_usage_reporting(
+        &self,
+        bucket: &str,
+        operation: QuotaOperation,
+        operation_size: u64,
+        force_usage_calculation: bool,
    ) -> Result<QuotaCheckResult, QuotaError> {
        let start_time = Instant::now();
        let quota_config = self.get_quota_config(bucket).await?;
@@ -43,7 +55,11 @@ impl QuotaChecker {
        // If no quota limit is set, allow operation
        let quota_limit = match quota_config.quota {
            None => {
-                let current_usage = self.get_real_time_usage(bucket).await?;
+                let current_usage = if force_usage_calculation {
+                    Some(self.get_real_time_usage(bucket).await?)
+                } else {
+                    None // Skip expensive usage calculation when no quota and not forced for performance
+                };
                return Ok(QuotaCheckResult {
                    allowed: true,
                    current_usage,
@@ -58,12 +74,12 @@ impl QuotaChecker {
        let current_usage = self.get_real_time_usage(bucket).await?;

        let expected_usage = match operation {
-            QuotaOperation::PutObject | QuotaOperation::CopyObject => current_usage + operation_size,
+            QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => current_usage + operation_size,
            QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size),
        };

        let allowed = match operation {
-            QuotaOperation::PutObject | QuotaOperation::CopyObject => {
+            QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
                quota_config.check_operation_allowed(current_usage, operation_size)
            }
            QuotaOperation::DeleteObject => true,
@@ -84,7 +100,7 @@ impl QuotaChecker {

        let result = QuotaCheckResult {
            allowed,
-            current_usage,
+            current_usage: Some(current_usage),
            quota_limit: Some(quota_limit),
            operation_size,
            remaining,
@@ -165,7 +181,7 @@ mod tests {
    async fn test_quota_check_no_limit() {
        let result = QuotaCheckResult {
            allowed: true,
-            current_usage: 0,
+            current_usage: None,
            quota_limit: None,
            operation_size: 1024,
            remaining: None,
@@ -80,7 +80,8 @@ impl BucketQuota {
 #[derive(Debug)]
 pub struct QuotaCheckResult {
    pub allowed: bool,
-    pub current_usage: u64,
+    /// current_usage: None when skipped for performance (no quota configured)
+    pub current_usage: Option<u64>,
    /// quota_limit: None means unlimited
    pub quota_limit: Option<u64>,
    pub operation_size: u64,
@@ -90,6 +91,7 @@ pub struct QuotaCheckResult {
 #[derive(Debug)]
 pub enum QuotaOperation {
    PutObject,
+    PostObject,
    CopyObject,
    DeleteObject,
 }
@@ -782,7 +782,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
            }

            // Generate random duration between 0 and 1 minute
-            use rand::Rng;
+            use rand::RngExt;
            let duration_millis = rand::rng().random_range(0..60_000);
            let mut duration = Duration::from_millis(duration_millis);

@@ -977,6 +977,10 @@ pub async fn init_background_replication<S: StorageAPI>(storage: Arc<S>) {
    assert!(GLOBAL_REPLICATION_POOL.get().is_some());
 }

+pub fn get_global_replication_pool() -> Option<Arc<DynReplicationPool>> {
+    GLOBAL_REPLICATION_POOL.get().cloned()
+}
+
 pub async fn schedule_replication<S: StorageAPI>(oi: ObjectInfo, o: Arc<S>, dsc: ReplicateDecision, op_type: ReplicationType) {
    let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
    let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+use crate::bucket::bandwidth::reader::{BucketOptions, MonitorReaderOptions, MonitoredReader};
 use crate::bucket::bucket_target_sys::{
    AdvancedPutOptions, BucketTargetSys, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
 };
@@ -28,15 +29,21 @@ use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_fo
 use crate::event::name::EventName;
 use crate::event_notification::{EventArgs, send_event};
 use crate::global::GLOBAL_LocalNodeName;
-use crate::store_api::{DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
+use crate::global::get_global_bucket_monitor;
+use crate::set_disk::get_lock_acquire_timeout;
+use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
 use crate::{StorageAPI, new_object_layer_fn};
 use aws_sdk_s3::error::SdkError;
 use aws_sdk_s3::operation::head_object::HeadObjectOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
+use aws_smithy_types::body::SdkBody;
 use byteorder::ByteOrder;
 use futures::future::join_all;
+use futures::stream::StreamExt;
 use http::HeaderMap;
+use http_body::Frame;
+use http_body_util::StreamBody;
 use regex::Regex;
 use rustfs_filemeta::{
    MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATION_RESET, ReplicateDecision, ReplicateObjectInfo,
@@ -60,10 +67,11 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
-use tokio::io::{AsyncRead, AsyncReadExt};
+use tokio::io::AsyncRead;
 use tokio::sync::RwLock;
 use tokio::task::JoinSet;
 use tokio::time::Duration as TokioDuration;
+use tokio_util::io::ReaderStream;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};

@@ -73,6 +81,8 @@ const RESYNC_META_FORMAT: u16 = 1;
 const RESYNC_META_VERSION: u16 = 1;
 const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60);

+static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new();
+
 #[derive(Debug, Clone, Default)]
 pub struct ResyncOpts {
    pub bucket: String,
@@ -1192,6 +1202,11 @@ pub async fn must_replicate(bucket: &str, object: &str, mopts: MustReplicateOpti
 }

 pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo, storage: Arc<S>) {
+    if dobj.delete_object.force_delete {
+        replicate_force_delete_to_targets(&dobj, storage).await;
+        return;
+    }
+
    let bucket = dobj.bucket.clone();
    let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
        Some(version_id.to_owned())
@@ -1273,7 +1288,58 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
        }
    };

-    //TODO: nslock
+    let ns_lock = match storage
+        .new_ns_lock(&bucket, format!("/[replicate]/{}", dobj.delete_object.object_name).as_str())
+        .await
+    {
+        Ok(ns_lock) => ns_lock,
+        Err(e) => {
+            warn!(
+                "failed to get ns lock for bucket:{} object:{} error:{}",
+                bucket, dobj.delete_object.object_name, e
+            );
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: dobj.delete_object.object_name.clone(),
+                    version_id,
+                    delete_marker: dobj.delete_object.delete_marker,
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };
+
+    let _lock_guard = match ns_lock.get_write_lock(get_lock_acquire_timeout()).await {
+        Ok(lock_guard) => lock_guard,
+        Err(e) => {
+            warn!(
+                "failed to get write lock for bucket:{} object:{} error:{}",
+                bucket, dobj.delete_object.object_name, e
+            );
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: dobj.delete_object.object_name.clone(),
+                    version_id,
+                    delete_marker: dobj.delete_object.delete_marker,
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };

    // Initialize replicated infos
    let mut rinfos = ReplicatedInfos {
@@ -1379,7 +1445,7 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
        dobj.delete_object.version_id.map(|v| v.to_string()),
    );
    if replication_status != prev_status {
-        drs.replica_timestamp = Some(OffsetDateTime::now_utc());
+        drs.replication_timestamp = Some(OffsetDateTime::now_utc());
    }

    let event_name = if replication_status == ReplicationStatusType::Completed {
@@ -1429,6 +1495,189 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
    }
 }

+async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
+    let bucket = &dobj.bucket;
+    let object_name = &dobj.delete_object.object_name;
+
+    let rcfg = match get_replication_config(bucket).await {
+        Ok(Some(config)) => config,
+        Ok(None) => {
+            warn!("replicate force-delete: no replication config for bucket:{}", bucket);
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object_name.clone(),
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+        Err(err) => {
+            warn!("replicate force-delete: replication config error bucket:{} error:{}", bucket, err);
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object_name.clone(),
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };
+
+    let ns_lock = match storage
+        .new_ns_lock(bucket, format!("/[replicate]/{}", object_name).as_str())
+        .await
+    {
+        Ok(ns_lock) => ns_lock,
+        Err(e) => {
+            warn!(
+                "replicate force-delete: failed to get ns lock bucket:{} object:{} error:{}",
+                bucket, object_name, e
+            );
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object_name.clone(),
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };
+
+    let _lock_guard = match ns_lock.get_write_lock(get_lock_acquire_timeout()).await {
+        Ok(guard) => guard,
+        Err(e) => {
+            warn!(
+                "replicate force-delete: failed to get write lock bucket:{} object:{} error:{}",
+                bucket, object_name, e
+            );
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object_name.clone(),
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };
+
+    let tgt_arns = if !dobj.target_arn.is_empty() {
+        vec![dobj.target_arn.clone()]
+    } else {
+        rcfg.filter_target_arns(&ObjectOpts {
+            name: object_name.clone(),
+            ..Default::default()
+        })
+    };
+
+    let mut join_set = JoinSet::new();
+
+    for arn in tgt_arns {
+        let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &arn).await else {
+            warn!("replicate force-delete: failed to get target client bucket:{} arn:{}", bucket, arn);
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object_name.clone(),
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            continue;
+        };
+
+        let bucket = bucket.clone();
+        let object_name = object_name.clone();
+
+        join_set.spawn(async move {
+            if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
+                error!("replicate force-delete: target offline bucket:{} arn:{}", bucket, tgt_client.arn);
+                send_event(EventArgs {
+                    event_name: EventName::ObjectReplicationFailed.as_ref().to_string(),
+                    bucket_name: bucket.clone(),
+                    object: ObjectInfo {
+                        bucket: bucket.clone(),
+                        name: object_name.clone(),
+                        ..Default::default()
+                    },
+                    user_agent: "Internal: [Replication]".to_string(),
+                    host: GLOBAL_LocalNodeName.to_string(),
+                    ..Default::default()
+                });
+                return;
+            }
+
+            if let Err(e) = tgt_client
+                .remove_object(
+                    &tgt_client.bucket,
+                    &object_name,
+                    None,
+                    RemoveObjectOptions {
+                        force_delete: true,
+                        governance_bypass: false,
+                        replication_delete_marker: false,
+                        replication_mtime: None,
+                        replication_status: ReplicationStatusType::Replica,
+                        replication_request: true,
+                        replication_validity_check: false,
+                    },
+                )
+                .await
+            {
+                error!(
+                    "replicate force-delete failed bucket:{} object:{} arn:{} error:{}",
+                    bucket, object_name, tgt_client.arn, e
+                );
+                send_event(EventArgs {
+                    event_name: EventName::ObjectReplicationFailed.as_ref().to_string(),
+                    bucket_name: bucket.clone(),
+                    object: ObjectInfo {
+                        bucket: bucket.clone(),
+                        name: object_name.clone(),
+                        ..Default::default()
+                    },
+                    user_agent: "Internal: [Replication]".to_string(),
+                    host: GLOBAL_LocalNodeName.to_string(),
+                    ..Default::default()
+                });
+            }
+        });
+    }
+
+    while let Some(result) = join_set.join_next().await {
+        if let Err(e) = result {
+            error!("replicate force-delete task panicked: {}", e);
+        }
+    }
+}
+
 async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
    let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
        version_id.to_owned()
@@ -1521,6 +1770,13 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
        }
    }

+    if rinfo.replication_status == ReplicationStatusType::Completed
+        && !tgt_client.reset_id.is_empty()
+        && dobj.op_type == ReplicationType::ExistingObject
+    {
+        rinfo.resync_timestamp = format!("{};{}", OffsetDateTime::now_utc().format(&Rfc3339).unwrap(), tgt_client.reset_id);
+    }
+
    rinfo
 }

@@ -1621,8 +1877,13 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
    let mut object_info = roi.to_object_info();

    if roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced() {
+        let mut eval_metadata = HashMap::new();
+        if let Some(ref s) = new_replication_internal {
+            eval_metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}replication-status"), s.clone());
+        }
        let popts = ObjectOptions {
            version_id: roi.version_id.map(|v| v.to_string()),
+            eval_metadata: Some(eval_metadata),
            ..Default::default()
        };

@@ -1704,20 +1965,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
        let versioned = BucketVersioningSys::prefix_enabled(&bucket, &object).await;
        let version_suspended = BucketVersioningSys::prefix_suspended(&bucket, &object).await;

+        let obj_opts = ObjectOptions {
+            version_id: self.version_id.map(|v| v.to_string()),
+            version_suspended,
+            versioned,
+            replication_request: true,
+            ..Default::default()
+        };
+
        let mut gr = match storage
-            .get_object_reader(
-                &bucket,
-                &object,
-                None,
-                HeaderMap::new(),
-                &ObjectOptions {
-                    version_id: self.version_id.map(|v| v.to_string()),
-                    version_suspended,
-                    versioned,
-                    replication_request: true,
-                    ..Default::default()
-                },
-            )
+            .get_object_reader(&bucket, &object, None, HeaderMap::new(), &obj_opts)
            .await
        {
            Ok(gr) => gr,
@@ -1772,6 +2029,40 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
            return rinfo;
        }

+        let mut replication_action = replication_action;
+        match tgt_client
+            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
+            .await
+        {
+            Ok(oi) => {
+                replication_action = get_replication_action(&object_info, &oi, self.op_type);
+                if replication_action == ReplicationAction::None {
+                    rinfo.replication_status = ReplicationStatusType::Completed;
+                    rinfo.replication_resynced = true;
+                    rinfo.replication_action = ReplicationAction::None;
+                    rinfo.size = size;
+                    return rinfo;
+                }
+            }
+            Err(e) => {
+                if let Some(se) = e.as_service_error() {
+                    if !se.is_not_found() {
+                        rinfo.error = Some(e.to_string());
+                        warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
+                        return rinfo;
+                    }
+                } else if e.raw_response().is_some_and(|resp| resp.status().as_u16() == 404) {
+                    // Some HEAD Object 404 responses are surfaced by the AWS SDK as `response error`
+                    // instead of `service error (NotFound)`. Treat raw HTTP 404 as object-not-found
+                    // so replication can proceed with PUT.
+                } else {
+                    rinfo.error = Some(e.to_string());
+                    warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
+                    return rinfo;
+                }
+            }
+        }
+
        rinfo.replication_status = ReplicationStatusType::Completed;
        rinfo.replication_resynced = true;
        rinfo.size = size;
@@ -1796,40 +2087,36 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
            }
        };

-        // TODO:bandwidth
-
        if let Some(err) = if is_multipart {
-            replicate_object_with_multipart(tgt_client.clone(), &tgt_client.bucket, &object, gr.stream, &object_info, put_opts)
-                .await
-                .err()
+            drop(gr);
+            replicate_object_with_multipart(MultipartReplicationContext {
+                storage: storage.clone(),
+                cli: tgt_client.clone(),
+                src_bucket: &bucket,
+                dst_bucket: &tgt_client.bucket,
+                object: &object,
+                object_info: &object_info,
+                obj_opts: &obj_opts,
+                arn: &rinfo.arn,
+                put_opts,
+            })
+            .await
+            .err()
        } else {
-            // TODO: use stream
-            let body = match gr.read_all().await {
-                Ok(body) => body,
-                Err(e) => {
-                    rinfo.replication_status = ReplicationStatusType::Failed;
-                    rinfo.error = Some(e.to_string());
-                    warn!("failed to read object for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
-                    send_event(EventArgs {
-                        event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
-                        bucket_name: bucket.clone(),
-                        object: object_info.clone(),
-                        host: GLOBAL_LocalNodeName.to_string(),
-                        user_agent: "Internal: [Replication]".to_string(),
-                        ..Default::default()
-                    });
-                    return rinfo;
-                }
-            };
-            let reader = ByteStream::from(body);
+            gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
+            let byte_stream = async_read_to_bytestream(gr.stream);
            tgt_client
-                .put_object(&tgt_client.bucket, &object, size, reader, &put_opts)
+                .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                .await
                .map_err(|e| std::io::Error::other(e.to_string()))
                .err()
        } {
            rinfo.replication_status = ReplicationStatusType::Failed;
            rinfo.error = Some(err.to_string());
+            warn!(
+                "replication put_object failed src_bucket={} dest_bucket={} object={} err={:?}",
+                bucket, tgt_client.bucket, object, err
+            );

            // TODO: check offline
            return rinfo;
@@ -1875,20 +2162,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
        let versioned = BucketVersioningSys::prefix_enabled(&bucket, &object).await;
        let version_suspended = BucketVersioningSys::prefix_suspended(&bucket, &object).await;

+        let obj_opts = ObjectOptions {
+            version_id: self.version_id.map(|v| v.to_string()),
+            version_suspended,
+            versioned,
+            replication_request: true,
+            ..Default::default()
+        };
+
        let mut gr = match storage
-            .get_object_reader(
-                &bucket,
-                &object,
-                None,
-                HeaderMap::new(),
-                &ObjectOptions {
-                    version_id: self.version_id.map(|v| v.to_string()),
-                    version_suspended,
-                    versioned,
-                    replication_request: true,
-                    ..Default::default()
-                },
-            )
+            .get_object_reader(&bucket, &object, None, HeaderMap::new(), &obj_opts)
            .await
        {
            Ok(gr) => gr,
@@ -2084,39 +2367,27 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                    return rinfo;
                }
            };
+
            if let Some(err) = if is_multipart {
-                replicate_object_with_multipart(
-                    tgt_client.clone(),
-                    &tgt_client.bucket,
-                    &object,
-                    gr.stream,
-                    &object_info,
+                drop(gr);
+                replicate_object_with_multipart(MultipartReplicationContext {
+                    storage: storage.clone(),
+                    cli: tgt_client.clone(),
+                    src_bucket: &bucket,
+                    dst_bucket: &tgt_client.bucket,
+                    object: &object,
+                    object_info: &object_info,
+                    obj_opts: &obj_opts,
+                    arn: &rinfo.arn,
                    put_opts,
-                )
+                })
                .await
                .err()
            } else {
-                let body = match gr.read_all().await {
-                    Ok(body) => body,
-                    Err(e) => {
-                        rinfo.replication_status = ReplicationStatusType::Failed;
-                        rinfo.error = Some(e.to_string());
-                        warn!("failed to read object for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
-                        send_event(EventArgs {
-                            event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
-                            bucket_name: bucket.clone(),
-                            object: object_info,
-                            host: GLOBAL_LocalNodeName.to_string(),
-                            user_agent: "Internal: [Replication]".to_string(),
-                            ..Default::default()
-                        });
-                        rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
-                        return rinfo;
-                    }
-                };
-                let reader = ByteStream::from(body);
+                gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
+                let byte_stream = async_read_to_bytestream(gr.stream);
                tgt_client
-                    .put_object(&tgt_client.bucket, &object, size, reader, &put_opts)
+                    .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                    .await
                    .map_err(|e| std::io::Error::other(e.to_string()))
                    .err()
@@ -2170,6 +2441,63 @@ static STANDARD_HEADERS: &[&str] = &[
    headers::AMZ_SERVER_SIDE_ENCRYPTION,
 ];

+fn calc_put_object_header_size(put_opts: &PutObjectOptions) -> usize {
+    let mut header_size: usize = 0;
+    for (key, value) in put_opts.header().iter() {
+        header_size += key.as_str().len();
+        header_size += value.as_bytes().len();
+        // Account for HTTP header formatting: ": " (2 bytes) and "\r\n" (2 bytes)
+        header_size += 4;
+    }
+    header_size
+}
+
+fn wrap_with_bandwidth_monitor_with_header(
+    stream: Box<dyn AsyncRead + Unpin + Send + Sync>,
+    bucket: &str,
+    arn: &str,
+    header_size: usize,
+) -> Box<dyn AsyncRead + Unpin + Send + Sync> {
+    if let Some(monitor) = get_global_bucket_monitor() {
+        Box::new(MonitoredReader::new(
+            monitor,
+            stream,
+            MonitorReaderOptions {
+                bucket_options: BucketOptions {
+                    name: bucket.to_string(),
+                    replication_arn: arn.to_string(),
+                },
+                header_size,
+            },
+        ))
+    } else {
+        WARNED_MONITOR_UNINIT.call_once(|| {
+            warn!(
+                "Global bucket monitor uninitialized; proceeding with unthrottled replication (bandwidth limits will be ignored)"
+            )
+        });
+        stream
+    }
+}
+
+fn wrap_with_bandwidth_monitor(
+    stream: Box<dyn AsyncRead + Unpin + Send + Sync>,
+    put_opts: &PutObjectOptions,
+    bucket: &str,
+    arn: &str,
+) -> Box<dyn AsyncRead + Unpin + Send + Sync> {
+    let header_size = calc_put_object_header_size(put_opts);
+    wrap_with_bandwidth_monitor_with_header(stream, bucket, arn, header_size)
+}
+
+fn async_read_to_bytestream(reader: impl AsyncRead + Send + Sync + Unpin + 'static) -> ByteStream {
+    // Non-retryable: SDK-level retries are not supported for streaming bodies.
+    // Replication-level retry handles failures at a higher layer.
+    let stream = ReaderStream::new(reader);
+    let body = StreamBody::new(stream.map(|r| r.map(Frame::data)));
+    ByteStream::new(SdkBody::from_body_1_x(body))
+}
+
 fn is_standard_header(k: &str) -> bool {
    STANDARD_HEADERS.iter().any(|h| h.eq_ignore_ascii_case(k))
 }
@@ -2393,17 +2721,56 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
    Ok((put_op, is_multipart))
 }

-async fn replicate_object_with_multipart(
+fn part_range_spec_from_actual_size(offset: i64, part_size: i64) -> std::io::Result<(HTTPRangeSpec, i64)> {
+    if offset < 0 {
+        return Err(std::io::Error::other("invalid part offset"));
+    }
+    if part_size <= 0 {
+        return Err(std::io::Error::other(format!("invalid part size {part_size}")));
+    }
+    let end = offset
+        .checked_add(part_size - 1)
+        .ok_or_else(|| std::io::Error::other("part range overflow"))?;
+    let next_offset = end
+        .checked_add(1)
+        .ok_or_else(|| std::io::Error::other("part offset overflow"))?;
+    Ok((
+        HTTPRangeSpec {
+            is_suffix_length: false,
+            start: offset,
+            end,
+        },
+        next_offset,
+    ))
+}
+
+struct MultipartReplicationContext<'a, S: StorageAPI> {
+    storage: Arc<S>,
    cli: Arc<TargetClient>,
-    bucket: &str,
-    object: &str,
-    reader: Box<dyn AsyncRead + Unpin + Send + Sync>,
-    object_info: &ObjectInfo,
-    opts: PutObjectOptions,
-) -> std::io::Result<()> {
+    src_bucket: &'a str,
+    dst_bucket: &'a str,
+    object: &'a str,
+    object_info: &'a ObjectInfo,
+    obj_opts: &'a ObjectOptions,
+    arn: &'a str,
+    put_opts: PutObjectOptions,
+}
+
+async fn replicate_object_with_multipart<S: StorageAPI>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
+    let MultipartReplicationContext {
+        storage,
+        cli,
+        src_bucket,
+        dst_bucket,
+        object,
+        object_info,
+        obj_opts,
+        arn,
+        put_opts,
+    } = ctx;
    let mut attempts = 1;
    let upload_id = loop {
-        match cli.create_multipart_upload(bucket, object, &opts).await {
+        match cli.create_multipart_upload(dst_bucket, object, &put_opts).await {
            Ok(id) => {
                break id;
            }
@@ -2422,19 +2789,30 @@ async fn replicate_object_with_multipart(

    let mut uploaded_parts: Vec<CompletedPart> = Vec::new();

-    let mut reader = reader;
+    let mut header_size = calc_put_object_header_size(&put_opts);
+    let mut offset: i64 = 0;
    for part_info in object_info.parts.iter() {
-        let mut chunk = vec![0u8; part_info.actual_size as usize];
-        AsyncReadExt::read_exact(&mut *reader, &mut chunk).await?;
+        let part_size = part_info.actual_size;
+        let (range_spec, next_offset) = part_range_spec_from_actual_size(offset, part_size)?;
+        offset = next_offset;
+
+        let part_reader = storage
+            .get_object_reader(src_bucket, object, Some(range_spec), HeaderMap::new(), obj_opts)
+            .await
+            .map_err(|e| std::io::Error::other(e.to_string()))?;
+
+        let part_stream = wrap_with_bandwidth_monitor_with_header(part_reader.stream, src_bucket, arn, header_size);
+        header_size = 0;
+        let byte_stream = async_read_to_bytestream(part_stream);

        let object_part = cli
            .put_object_part(
-                bucket,
+                dst_bucket,
                object,
                &upload_id,
                part_info.number as i32,
-                part_info.actual_size,
-                ByteStream::from(chunk),
+                part_size,
+                byte_stream,
                &PutObjectPartOptions { ..Default::default() },
            )
            .await
@@ -2462,7 +2840,7 @@ async fn replicate_object_with_multipart(
    );

    cli.complete_multipart_upload(
-        bucket,
+        dst_bucket,
        object,
        &upload_id,
        uploaded_parts,
@@ -2583,3 +2961,22 @@ fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: Rep

    ReplicationAction::None
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_part_range_spec_from_actual_size() {
+        let (rs, next) = part_range_spec_from_actual_size(0, 10).unwrap();
+        assert_eq!(rs.start, 0);
+        assert_eq!(rs.end, 9);
+        assert_eq!(next, 10);
+    }
+
+    #[test]
+    fn test_part_range_spec_rejects_non_positive() {
+        assert!(part_range_spec_from_actual_size(0, 0).is_err());
+        assert!(part_range_spec_from_actual_size(0, -1).is_err());
+    }
+}
@@ -193,10 +193,6 @@ pub fn is_valid_object_name(object: &str) -> bool {
        return false;
    }

-    if object.ends_with(SLASH_SEPARATOR) {
-        return false;
-    }
-
    is_valid_object_prefix(object)
 }

@@ -379,10 +375,10 @@ mod tests {
        // Invalid cases - empty string
        assert!(!is_valid_object_name(""));

-        // Invalid cases - ends with slash (object names cannot end with slash)
-        assert!(!is_valid_object_name("object/"));
-        assert!(!is_valid_object_name("path/to/file/"));
-        assert!(!is_valid_object_name("ends/with/slash/"));
+        // Valid cases - trailing slash is allowed (empty object with trailing slash)
+        assert!(is_valid_object_name("object/"));
+        assert!(is_valid_object_name("path/to/file/"));
+        assert!(is_valid_object_name("ends/with/slash/"));

        // Invalid cases - bad path components (inherited from is_valid_object_prefix)
        assert!(!is_valid_object_name("."));
@@ -36,10 +36,7 @@ impl VersioningApi for VersioningConfiguration {
            return true;
        }

-        if let Some(exclude_folders) = self.exclude_folders
-            && exclude_folders
-            && prefix.ends_with('/')
-        {
+        if self.exclude_folders.unwrap_or(false) && prefix.ends_with('/') {
            return false;
        }

--- a/Show More
+++ b/Show More