chore(deps): update flake.lock (#2165)

Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: 马登山 <Cxymds@qq.com>

.config/make/tests.mak

@@ -1,5 +1,7 @@
 ## —— Tests and e2e test ---------------------------------------------------------------------------
 
+TEST_THREADS ?= 1
+
 .PHONY: test
 test: core-deps test-deps ## Run all tests
 	@echo "🧪 Running tests..."
@@ -7,7 +9,7 @@ test: core-deps test-deps ## Run all tests
 		cargo nextest run --all --exclude e2e_test; \
 	else \
 		echo "ℹ️ cargo-nextest not found; falling back to 'cargo test'"; \
-		cargo test --workspace --exclude e2e_test -- --nocapture; \
+		cargo test --workspace --exclude e2e_test -- --nocapture --test-threads="$(TEST_THREADS)"; \
 	fi
 	cargo test --all --doc
 
@@ -17,4 +19,4 @@ e2e-server: ## Run e2e-server tests
 
 .PHONY: probe-e2e
 probe-e2e: ## Probe e2e tests
-	sh $(shell pwd)/scripts/probe.sh
+	sh $(shell pwd)/scripts/probe.sh

.docker/README.md

@@ -1,261 +1,131 @@
-# RustFS Docker Images
+# RustFS Docker Infrastructure
 
-This directory contains Docker configuration files and supporting infrastructure for building and running RustFS container images.
+This directory contains the complete Docker infrastructure for building, deploying, and monitoring RustFS. It provides ready-to-use configurations for development, testing, and production-grade observability.
 
-## 📁 Directory Structure
+## 📂 Directory Structure
 
-```
-rustfs/
-├── Dockerfile           # Production image (Alpine + pre-built binaries)
-├── Dockerfile.source    # Development image (Debian + source build)
-├── docker-buildx.sh     # Multi-architecture build script
-├── Makefile             # Build automation with simplified commands
-└── .docker/             # Supporting infrastructure
-    ├── observability/   # Monitoring and observability configs
-    ├── compose/         # Docker Compose configurations
-    ├── mqtt/            # MQTT broker configs
-    └── openobserve-otel/ # OpenObserve + OpenTelemetry configs
-```
+| Directory | Description | Status |
+| :--- | :--- | :--- |
+| **[`observability/`](observability/README.md)** | **[RECOMMENDED]** Full-stack observability (Prometheus, Grafana, Tempo, Loki). | ✅ Production-Ready |
+| **[`compose/`](compose/README.md)** | Specialized setups (e.g., 4-node distributed cluster testing). | ⚠️ Testing Only |
+| **[`mqtt/`](mqtt/README.md)** | EMQX Broker configuration for MQTT integration testing. | 🧪 Development |
+| **[`openobserve-otel/`](openobserve-otel/README.md)** | Alternative lightweight observability stack using OpenObserve. | 🔄 Alternative |
 
-## 🎯 Image Variants
+---
 
-### Core Images
+## 📄 Root Directory Files
 
-| Image | Base OS | Build Method | Size | Use Case |
-|-------|---------|--------------|------|----------|
-| `production` (default) | Alpine 3.18 | GitHub Releases | Smallest | Production deployment |
-| `source` | Debian Bookworm | Source build | Medium | Custom builds with cross-compilation |
-| `dev` | Debian Bookworm | Development tools | Large | Interactive development |
+The following files in the project root are essential for Docker operations:
 
-## 🚀 Usage Examples
+### Build Scripts & Dockerfiles
 
-### Quick Start (Production)
+| File | Description | Usage |
+| :--- | :--- | :--- |
+| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
+| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
+| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
+| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
 
+### Docker Compose Configurations
+
+| File | Description | Usage |
+| :--- | :--- | :--- |
+| **`docker-compose.yml`** | **Main Development Setup**<br>Comprehensive setup with profiles for development, observability, and proxying. | `docker compose up -d`<br>`docker compose --profile observability up -d` |
+| **`docker-compose-simple.yml`** | **Quick Start Setup**<br>Minimal configuration running a single RustFS instance with 4 volumes. Perfect for first-time users. | `docker compose -f docker-compose-simple.yml up -d` |
+
+---
+
+## 🌟 Observability Stack (Recommended)
+
+Located in: [`.docker/observability/`](observability/README.md)
+
+We provide a comprehensive, industry-standard observability stack designed for deep insights into RustFS performance. This is the recommended setup for both development and production monitoring.
+
+### Components
+- **Metrics**: Prometheus (Collection) + Grafana (Visualization)
+- **Traces**: Tempo (Storage) + Jaeger (UI)
+- **Logs**: Loki
+- **Ingestion**: OpenTelemetry Collector
+
+### Key Features
+- **Full Persistence**: All metrics, logs, and traces are saved to Docker volumes, ensuring no data loss on restarts.
+- **Correlation**: Seamlessly jump between Logs, Traces, and Metrics in Grafana.
+- **High Performance**: Optimized configurations for batching, compression, and memory management.
+
+### Quick Start
 ```bash
-# Default production image (Alpine + GitHub Releases)
-docker run -p 9000:9000 rustfs/rustfs:latest
-
-# Specific version
-docker run -p 9000:9000 rustfs/rustfs:1.2.3
+cd .docker/observability
+docker compose up -d
 ```
 
-### Complete Tag Strategy Examples
+---
 
+## 🧪 Specialized Environments
+
+Located in: [`.docker/compose/`](compose/README.md)
+
+These configurations are tailored for specific testing scenarios that require complex topologies.
+
+### Distributed Cluster (4-Nodes)
+Simulates a real-world distributed environment with 4 RustFS nodes running locally.
 ```bash
-# Stable Releases
-docker run rustfs/rustfs:1.2.3           # Main version (production)
-docker run rustfs/rustfs:1.2.3-production # Explicit production variant
-docker run rustfs/rustfs:1.2.3-source   # Source build variant
-docker run rustfs/rustfs:latest          # Latest stable
-
-# Prerelease Versions
-docker run rustfs/rustfs:1.3.0-alpha.2  # Specific alpha version
-docker run rustfs/rustfs:alpha           # Latest alpha
-docker run rustfs/rustfs:beta            # Latest beta
-docker run rustfs/rustfs:rc              # Latest release candidate
-
-# Development Versions
-docker run rustfs/rustfs:dev             # Latest main branch development
-docker run rustfs/rustfs:dev-13e4a0b     # Specific commit
-docker run rustfs/rustfs:dev-latest      # Latest development
-docker run rustfs/rustfs:main-latest     # Main branch latest
+docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
 ```
 
-### Development Environment
-
+### Integrated Observability Test
+A self-contained environment running 4 RustFS nodes alongside the full observability stack. Useful for end-to-end testing of telemetry.
 ```bash
-# Quick setup using Makefile (recommended)
-make docker-dev-local     # Build development image locally
-make dev-env-start        # Start development container
-
-# Manual Docker commands
-docker run -it -v $(pwd):/workspace -p 9000:9000 rustfs/rustfs:latest-dev
-
-# Build from source locally
-docker build -f Dockerfile.source -t rustfs:custom .
-
-# Development with hot reload
-docker-compose up rustfs-dev
+docker compose -f .docker/compose/docker-compose.observability.yaml up -d
 ```
 
-## 🏗️ Build Arguments and Scripts
+---
 
-### Using Makefile Commands (Recommended)
+## 📡 MQTT Integration
 
-The easiest way to build images using simplified commands:
+Located in: [`.docker/mqtt/`](mqtt/README.md)
 
+Provides an EMQX broker for testing RustFS MQTT features.
+
+### Quick Start
 ```bash
-# Development images (build from source)
-make docker-dev-local             # Build for local use (single arch)
-make docker-dev                   # Build multi-arch (for CI/CD)
-make docker-dev-push REGISTRY=xxx # Build and push to registry
-
-# Production images (using pre-built binaries)
-make docker-buildx                # Build multi-arch production images
-make docker-buildx-push           # Build and push production images
-make docker-buildx-version VERSION=v1.0.0  # Build specific version
-
-# Development environment
-make dev-env-start                # Start development container
-make dev-env-stop                 # Stop development container
-make dev-env-restart              # Restart development container
-
-# Help
-make help-docker                  # Show all Docker-related commands
+cd .docker/mqtt
+docker compose up -d
 ```
+- **Dashboard**: [http://localhost:18083](http://localhost:18083) (Default: `admin` / `public`)
+- **MQTT Port**: `1883`
 
-### Using docker-buildx.sh (Advanced)
+---
 
-For direct script usage and advanced scenarios:
+## 👁️ Alternative: OpenObserve
 
+Located in: [`.docker/openobserve-otel/`](openobserve-otel/README.md)
+
+For users preferring a lightweight, all-in-one solution, we support OpenObserve. It combines logs, metrics, and traces into a single binary and UI.
+
+### Quick Start
 ```bash
-# Build latest version for all architectures
-./docker-buildx.sh
-
-# Build and push to registry
-./docker-buildx.sh --push
-
-# Build specific version
-./docker-buildx.sh --release v1.2.3
-
-# Build and push specific version
-./docker-buildx.sh --release v1.2.3 --push
+cd .docker/openobserve-otel
+docker compose up -d
 ```
 
-### Manual Docker Builds
+---
 
-All images support dynamic version selection:
+## 🔧 Common Operations
 
+### Cleaning Up
+To stop all containers and remove volumes (**WARNING**: deletes all persisted data):
 ```bash
-# Build production image with latest release
-docker build --build-arg RELEASE="latest" -t rustfs:latest .
-
-# Build from source with specific target
-docker build -f Dockerfile.source \
-  --build-arg TARGETPLATFORM="linux/amd64" \
-  -t rustfs:source .
-
-# Development build
-docker build -f Dockerfile.source -t rustfs:dev .
+docker compose down -v
 ```
 
-## 🔧 Binary Download Sources
-
-### Unified GitHub Releases
-
-The production image downloads from GitHub Releases for reliability and transparency:
-
-- ✅ **production** → GitHub Releases API with automatic latest detection
-- ✅ **Checksum verification** → SHA256SUMS validation when available
-- ✅ **Multi-architecture** → Supports amd64 and arm64
-
-### Source Build
-
-The source variant compiles from source code with advanced features:
-
-- 🔧 **Cross-compilation** → Supports multiple target platforms via `TARGETPLATFORM`
-- ⚡ **Build caching** → sccache for faster compilation
-- 🎯 **Optimized builds** → Release optimizations with LTO and symbol stripping
-
-## 📋 Architecture Support
-
-All variants support multi-architecture builds:
-
-- **linux/amd64** (x86_64)
-- **linux/arm64** (aarch64)
-
-Architecture is automatically detected during build using Docker's `TARGETARCH` build argument.
-
-## 🔐 Security Features
-
-- **Checksum Verification**: Production image verifies SHA256SUMS when available
-- **Non-root User**: All images run as user `rustfs` (UID 1000)
-- **Minimal Runtime**: Production image only includes necessary dependencies
-- **Secure Defaults**: No hardcoded credentials or keys
-
-## 🛠️ Development Workflow
-
-### Quick Start with Makefile (Recommended)
-
+### Viewing Logs
+To follow logs for a specific service:
 ```bash
-# 1. Start development environment
-make dev-env-start
-
-# 2. Your development container is now running with:
-#    - Port 9000 exposed for RustFS
-#    - Port 9010 exposed for admin console
-#    - Current directory mounted as /workspace
-
-# 3. Stop when done
-make dev-env-stop
+docker compose logs -f [service_name]
 ```
 
-### Manual Development Setup
-
+### Checking Status
+To see the status of all running containers:
 ```bash
-# Build development image from source
-make docker-dev-local
-
-# Or use traditional Docker commands
-docker build -f Dockerfile.source -t rustfs:dev .
-
-# Run with development tools
-docker run -it -v $(pwd):/workspace -p 9000:9000 rustfs:dev bash
-
-# Or use docker-compose for complex setups
-docker-compose up rustfs-dev
+docker compose ps
 ```
-
-### Common Development Tasks
-
-```bash
-# Build and test locally
-make build                        # Build binary natively
-make docker-dev-local            # Build development Docker image
-make test                        # Run tests
-make fmt                         # Format code
-make clippy                      # Run linter
-
-# Get help
-make help                        # General help
-make help-docker                 # Docker-specific help
-make help-build                  # Build-specific help
-```
-
-## 🚀 CI/CD Integration
-
-The project uses GitHub Actions for automated multi-architecture Docker builds:
-
-### Automated Builds
-
-- **Tags**: Automatic builds triggered on version tags (e.g., `v1.2.3`)
-- **Main Branch**: Development builds with `dev-latest` and `main-latest` tags
-- **Pull Requests**: Test builds without registry push
-
-### Build Variants
-
-Each build creates three image variants:
-
-- `rustfs/rustfs:v1.2.3` (production - Alpine-based)
-- `rustfs/rustfs:v1.2.3-source` (source build - Debian-based)
-- `rustfs/rustfs:v1.2.3-dev` (development - Debian-based with tools)
-
-### Manual Builds
-
-Trigger custom builds via GitHub Actions:
-
-```bash
-# Use workflow_dispatch to build specific versions
-# Available options: latest, main-latest, dev-latest, v1.2.3, dev-abc123
-```
-
-## 📦 Supporting Infrastructure
-
-The `.docker/` directory contains supporting configuration files:
-
-- **observability/** - Prometheus, Grafana, OpenTelemetry configs
-- **compose/** - Multi-service Docker Compose setups
-- **mqtt/** - MQTT broker configurations
-- **openobserve-otel/** - Log aggregation and tracing setup
-
-See individual README files in each subdirectory for specific usage instructions.

.docker/compose/README.md

@@ -1,80 +1,44 @@
-# Docker Compose Configurations
+# Specialized Docker Compose Configurations
 
-This directory contains specialized Docker Compose configurations for different use cases.
+This directory contains specialized Docker Compose configurations for specific testing scenarios.
+
+## ⚠️ Important Note
+
+**For Observability:**
+We **strongly recommend** using the new, fully integrated observability stack located in `../observability/`. It provides a production-ready setup with Prometheus, Grafana, Tempo, Loki, and OpenTelemetry Collector, all with persistent storage and optimized configurations.
+
+The `docker-compose.observability.yaml` in this directory is kept for legacy reference or specific minimal testing needs but is **not** the primary recommended setup.
 
 ## 📁 Configuration Files
 
-This directory contains specialized Docker Compose configurations and their associated Dockerfiles, keeping related files organized together.
+### Cluster Testing
 
-### Main Configuration (Root Directory)
+- **`docker-compose.cluster.yaml`**
+  - **Purpose**: Simulates a 4-node RustFS distributed cluster.
+  - **Use Case**: Testing distributed storage logic, consensus, and failover.
+  - **Nodes**: 4 RustFS instances.
+  - **Storage**: Uses local HTTP endpoints.
 
-- **`../../docker-compose.yml`** - **Default Production Setup**
-  - Complete production-ready configuration
-  - Includes RustFS server + full observability stack
-  - Supports multiple profiles: `dev`, `observability`, `cache`, `proxy`
-  - Recommended for most users
+### Legacy / Minimal Observability
 
-### Specialized Configurations
-
-- **`docker-compose.cluster.yaml`** - **Distributed Testing**
-  - 4-node cluster setup for testing distributed storage
-  - Uses local compiled binaries
-  - Simulates multi-node environment
-  - Ideal for development and cluster testing
-
-- **`docker-compose.observability.yaml`** - **Observability Focus**
-  - Specialized setup for testing observability features
-  - Includes OpenTelemetry, Jaeger, Prometheus, Loki, Grafana
-  - Uses `../../Dockerfile.source` for builds
-  - Perfect for observability development
+- **`docker-compose.observability.yaml`**
+  - **Purpose**: A minimal observability setup.
+  - **Status**: **Deprecated**. Please use `../observability/docker-compose.yml` instead.
 
 ## 🚀 Usage Examples
 
-### Production Setup
-
-```bash
-# Start main service
-docker-compose up -d
-
-# Start with development profile
-docker-compose --profile dev up -d
-
-# Start with full observability
-docker-compose --profile observability up -d
-```
-
 ### Cluster Testing
 
-```bash
-# Build and start 4-node cluster (run from project root)
-cd .docker/compose
-docker-compose -f docker-compose.cluster.yaml up -d
-
-# Or run directly from project root
-docker-compose -f .docker/compose/docker-compose.cluster.yaml up -d
-```
-
-### Observability Testing
+To start a 4-node cluster for distributed testing:
 
 ```bash
-# Start observability-focused environment (run from project root)
-cd .docker/compose
-docker-compose -f docker-compose.observability.yaml up -d
-
-# Or run directly from project root
-docker-compose -f .docker/compose/docker-compose.observability.yaml up -d
+# From project root
+docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
 ```
 
-## 🔧 Configuration Overview
+### (Deprecated) Minimal Observability
 
-| Configuration | Nodes | Storage | Observability | Use Case |
-|---------------|-------|---------|---------------|----------|
-| **Main** | 1 | Volume mounts | Full stack | Production |
-| **Cluster** | 4 | HTTP endpoints | Basic | Testing |
-| **Observability** | 4 | Local data | Advanced | Development |
-
-## 📝 Notes
-
-- Always ensure you have built the required binaries before starting cluster tests
-- The main configuration is sufficient for most use cases
-- Specialized configurations are for specific testing scenarios
+```bash
+# From project root
+docker compose -f .docker/compose/docker-compose.observability.yaml up -d
+```

.docker/compose/docker-compose.observability.yaml

@@ -13,65 +13,126 @@
 # limitations under the License.
 
 services:
+  # --- Observability Stack ---
+
+  tempo-init:
+    image: busybox:latest
+    command: [ "sh", "-c", "chown -R 10001:10001 /var/tempo" ]
+    volumes:
+      - tempo-data:/var/tempo
+    user: root
+    networks:
+      - rustfs-network
+    restart: "no"
+
+  tempo:
+    image: grafana/tempo:latest
+    user: "10001"
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ../../.docker/observability/tempo.yaml:/etc/tempo.yaml:ro
+      - tempo-data:/var/tempo
+    ports:
+      - "3200:3200" # tempo
+      - "4317"      # otlp grpc
+      - "4318"      # otlp http
+    restart: unless-stopped
+    networks:
+      - rustfs-network
+
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.129.1
+    image: otel/opentelemetry-collector-contrib:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
-      - ../../.docker/observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
+      - ../../.docker/observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
     ports:
-      - 1888:1888
-      - 8888:8888
-      - 8889:8889
-      - 13133:13133
-      - 4317:4317
-      - 4318:4318
-      - 55679:55679
+      - "1888:1888"   # pprof
+      - "8888:8888"   # Prometheus metrics for Collector
+      - "8889:8889"   # Prometheus metrics for application indicators
+      - "13133:13133" # health check
+      - "4317:4317"   # OTLP gRPC
+      - "4318:4318"   # OTLP HTTP
+      - "55679:55679" # zpages
     networks:
       - rustfs-network
+    depends_on:
+      - tempo
+      - jaeger
+      - prometheus
+      - loki
+
   jaeger:
-    image: jaegertracing/jaeger:2.8.0
+    image: jaegertracing/jaeger:latest
     environment:
       - TZ=Asia/Shanghai
+      - SPAN_STORAGE_TYPE=badger
+      - BADGER_EPHEMERAL=false
+      - BADGER_DIRECTORY_VALUE=/badger/data
+      - BADGER_DIRECTORY_KEY=/badger/key
+      - COLLECTOR_OTLP_ENABLED=true
+    volumes:
+      - jaeger-data:/badger
     ports:
-      - "16686:16686"
-      - "14317:4317"
-      - "14318:4318"
+      - "16686:16686" # Web UI
+      - "14269:14269" # Admin/Metrics
     networks:
       - rustfs-network
+
   prometheus:
-    image: prom/prometheus:v3.4.2
+    image: prom/prometheus:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
-      - ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
     ports:
       - "9090:9090"
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.enable-otlp-receiver'
+      - '--web.enable-remote-write-receiver'
+      - '--enable-feature=promql-experimental-functions'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
     networks:
       - rustfs-network
+
   loki:
-    image: grafana/loki:3.5.1
+    image: grafana/loki:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
-      - ../../.docker/observability/loki-config.yaml:/etc/loki/local-config.yaml
+      - ../../.docker/observability/loki.yaml:/etc/loki/local-config.yaml:ro
+      - loki-data:/loki
     ports:
       - "3100:3100"
     command: -config.file=/etc/loki/local-config.yaml
     networks:
       - rustfs-network
+
   grafana:
-    image: grafana/grafana:12.0.2
+    image: grafana/grafana:latest
     ports:
       - "3000:3000" # Web UI
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_SECURITY_ADMIN_USER=admin
       - TZ=Asia/Shanghai
+      - GF_INSTALL_PLUGINS=grafana-pyroscope-datasource
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/home.json
     networks:
       - rustfs-network
     volumes:
       - ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro
       - ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+    depends_on:
+      - prometheus
+      - tempo
+      - loki
+
+  # --- RustFS Cluster ---
 
   node1:
     build:
@@ -86,9 +147,11 @@ services:
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
-      - "9001:9000" # Map port 9001 of the host to port 9000 of the container
+      - "9001:9000"
     networks:
       - rustfs-network
+    depends_on:
+      - otel-collector
 
   node2:
     build:
@@ -103,9 +166,11 @@ services:
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
-      - "9002:9000" # Map port 9002 of the host to port 9000 of the container
+      - "9002:9000"
     networks:
       - rustfs-network
+    depends_on:
+      - otel-collector
 
   node3:
     build:
@@ -120,9 +185,11 @@ services:
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
-      - "9003:9000" # Map port 9003 of the host to port 9000 of the container
+      - "9003:9000"
     networks:
       - rustfs-network
+    depends_on:
+      - otel-collector
 
   node4:
     build:
@@ -137,9 +204,17 @@ services:
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
-      - "9004:9000" # Map port 9004 of the host to port 9000 of the container
+      - "9004:9000"
     networks:
       - rustfs-network
+    depends_on:
+      - otel-collector
+
+volumes:
+  prometheus-data:
+  tempo-data:
+  loki-data:
+  jaeger-data:
 
 networks:
   rustfs-network:

.docker/mqtt/README.md

@@ -0,0 +1,30 @@
+# MQTT Broker (EMQX)
+
+This directory contains the configuration for running an EMQX MQTT broker, which can be used for testing RustFS's MQTT integration.
+
+## 🚀 Quick Start
+
+To start the EMQX broker:
+
+```bash
+docker compose up -d
+```
+
+## 📊 Access
+
+- **Dashboard**: [http://localhost:18083](http://localhost:18083)
+- **Default Credentials**: `admin` / `public`
+- **MQTT Port**: `1883`
+- **WebSocket Port**: `8083`
+
+## 🛠️ Configuration
+
+The `docker-compose.yml` file sets up a single-node EMQX instance.
+
+- **Persistence**: Data is not persisted by default (for testing).
+- **Network**: Uses the default bridge network.
+
+## 📝 Notes
+
+- This setup is intended for development and testing purposes.
+- For production deployments, please refer to the official [EMQX Documentation](https://www.emqx.io/docs/en/latest/).

.docker/nginx/nginx.conf

@@ -0,0 +1,82 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+worker_processes auto;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+    error_log /var/log/nginx/error.log warn;
+
+    sendfile on;
+    keepalive_timeout 65;
+
+    # RustFS Server Block
+    server {
+        listen 80;
+        server_name localhost;
+
+        # Redirect HTTP to HTTPS (optional, uncomment if SSL is configured)
+        # return 301 https://$host$request_uri;
+
+        location / {
+            proxy_pass http://rustfs:9000;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            # S3 specific headers
+            proxy_set_header X-Amz-Date $http_x_amz_date;
+            proxy_set_header Authorization $http_authorization;
+
+            # Disable buffering for large uploads
+            proxy_request_buffering off;
+            client_max_body_size 0;
+        }
+
+        location /rustfs/console {
+            proxy_pass http://rustfs:9001;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+    }
+
+    # SSL Configuration (Example)
+    # server {
+    #     listen 443 ssl;
+    #     server_name localhost;
+    #
+    #     ssl_certificate /etc/nginx/ssl/server.crt;
+    #     ssl_certificate_key /etc/nginx/ssl/server.key;
+    #
+    #     location / {
+    #         proxy_pass http://rustfs:9000;
+    #         ...
+    #     }
+    # }
+}

.docker/observability/.gitignore

@@ -0,0 +1,5 @@
+jaeger-data/*
+loki-data/*
+prometheus-data/*
+tempo-data/*
+grafana-data/*

.docker/observability/README.md

@@ -1,109 +1,85 @@
-# Observability
+# RustFS Observability Stack
 
-This directory contains the observability stack for the application. The stack is composed of the following components:
+This directory contains the comprehensive observability stack for RustFS, designed to provide deep insights into application performance, logs, and traces.
 
-- Prometheus v3.2.1
-- Grafana 11.6.0
-- Loki 3.4.2
-- Jaeger 2.4.0
-- Otel Collector 0.120.0 # 0.121.0 remove loki
+## Components
 
-## Prometheus
+The stack is composed of the following best-in-class open-source components:
 
-Prometheus is a monitoring and alerting toolkit. It scrapes metrics from instrumented jobs, either directly or via an
-intermediary push gateway for short-lived jobs. It stores all scraped samples locally and runs rules over this data to
-either aggregate and record new time series from existing data or generate alerts. Grafana or other API consumers can be
-used to visualize the collected data.
+- **Prometheus** (v2.53.1): The industry standard for metric collection and alerting.
+- **Grafana** (v11.1.0): The leading platform for observability visualization.
+- **Loki** (v3.1.0): A horizontally-scalable, highly-available, multi-tenant log aggregation system.
+- **Tempo** (v2.5.0): A high-volume, minimal dependency distributed tracing backend.
+- **Jaeger** (v1.59.0): Distributed tracing system (configured as a secondary UI/storage).
+- **OpenTelemetry Collector** (v0.104.0): A vendor-agnostic implementation for receiving, processing, and exporting telemetry data.
 
-## Grafana
+## Architecture
 
-Grafana is a multi-platform open-source analytics and interactive visualization web application. It provides charts,
-graphs, and alerts for the web when connected to supported data sources.
+1. **Telemetry Collection**: Applications send OTLP (OpenTelemetry Protocol) data (Metrics, Logs, Traces) to the **OpenTelemetry Collector**.
+2. **Processing & Exporting**: The Collector processes the data (batching, memory limiting) and exports it to the respective backends:
+    - **Traces** -> **Tempo** (Primary) & **Jaeger** (Secondary/Optional)
+    - **Metrics** -> **Prometheus** (via scraping the Collector's exporter)
+    - **Logs** -> **Loki**
+3. **Visualization**: **Grafana** connects to all backends (Prometheus, Tempo, Loki, Jaeger) to provide a unified dashboard experience.
 
-## Loki
+## Features
 
-Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. It is
-designed to be very cost-effective and easy to operate. It does not index the contents of the logs, but rather a set of
-labels for each log stream.
+- **Full Persistence**: All data (Metrics, Logs, Traces) is persisted to Docker volumes, ensuring no data loss on restart.
+- **Correlation**: Seamless navigation between Metrics, Logs, and Traces in Grafana.
+  - Jump from a Metric spike to relevant Traces.
+  - Jump from a Trace to relevant Logs.
+- **High Performance**: Optimized configurations for batching, compression, and memory management.
+- **Standardized Protocols**: Built entirely on OpenTelemetry standards.
 
-## Jaeger
+## Quick Start
 
-Jaeger is a distributed tracing system released as open source by Uber Technologies. It is used for monitoring and
-troubleshooting microservices-based distributed systems, including:
+### Prerequisites
 
-- Distributed context propagation
-- Distributed transaction monitoring
-- Root cause analysis
-- Service dependency analysis
-- Performance / latency optimization
+- Docker
+- Docker Compose
 
-## Otel Collector
+### Deploy
 
-The OpenTelemetry Collector offers a vendor-agnostic implementation on how to receive, process, and export telemetry
-data. It removes the need to run, operate, and maintain multiple agents/collectors in order to support open-source
-observability data formats (e.g. Jaeger, Prometheus, etc.) sending to one or more open-source or commercial back-ends.
-
-## How to use
-
-To deploy the observability stack, run the following command:
-
-- docker latest version
+Run the following command to start the entire stack:
 
 ```bash
-docker compose -f docker-compose.yml -f docker-compose.override.yml up -d
+docker compose up -d
 ```
 
-- docker compose v2.0.0 or before
+### Access Dashboards
+
+| Service        | URL                                              | Credentials       | Description                    |
+| :------------- | :----------------------------------------------- | :---------------- | :----------------------------- |
+| **Grafana**    | [http://localhost:3000](http://localhost:3000)   | `admin` / `admin` | Main visualization hub.        |
+| **Prometheus** | [http://localhost:9090](http://localhost:9090)   | -                 | Metric queries and status.     |
+| **Jaeger UI**  | [http://localhost:16686](http://localhost:16686) | -                 | Secondary trace visualization. |
+| **Tempo**      | [http://localhost:3200](http://localhost:3200)   | -                 | Tempo status/metrics.          |
+
+## Configuration
+
+### Data Persistence
+
+Data is stored in the following Docker volumes:
+
+- `prometheus-data`: Prometheus metrics
+- `tempo-data`: Tempo traces (WAL and Blocks)
+- `loki-data`: Loki logs (Chunks and Rules)
+- `jaeger-data`: Jaeger traces (Badger DB)
+
+To clear all data:
 
 ```bash
-docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d
+docker compose down -v
 ```
 
-To access the Grafana dashboard, navigate to `http://localhost:3000` in your browser. The default username and password
-are `admin` and `admin`, respectively.
-
-To access the Jaeger dashboard, navigate to `http://localhost:16686` in your browser.
-
-To access the Prometheus dashboard, navigate to `http://localhost:9090` in your browser.
-
-## How to stop
-
-To stop the observability stack, run the following command:
-
-```bash
-docker compose -f docker-compose.yml -f docker-compose.override.yml down
-```
-
-## How to remove data
-
-To remove the data generated by the observability stack, run the following command:
-
-```bash
-docker compose -f docker-compose.yml -f docker-compose.override.yml down -v
-```
-
-## How to configure
-
-To configure the observability stack, modify the `docker-compose.override.yml` file. The file contains the following
-
-```yaml
-services:
-  prometheus:
-    environment:
-      - PROMETHEUS_CONFIG_FILE=/etc/prometheus/prometheus.yml
-    volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
-
-  grafana:
-    environment:
-      - GF_SECURITY_ADMIN_PASSWORD=admin
-    volumes:
-      - ./grafana/provisioning:/etc/grafana/provisioning
-```
-
-The `prometheus` service mounts the `prometheus.yml` file to `/etc/prometheus/prometheus.yml`. The `grafana` service
-mounts the `grafana/provisioning` directory to `/etc/grafana/provisioning`. You can modify these files to configure the
-observability stack.
+### Customization
 
+- **Prometheus**: Edit `prometheus.yml` to add scrape targets or alerting rules.
+- **Grafana**: Dashboards and datasources are provisioned from the `grafana/` directory.
+- **Collector**: Edit `otel-collector-config.yaml` to modify pipelines, processors, or exporters.
 
+## Troubleshooting
 
+- **Service Health**: Check the health of services using `docker compose ps`.
+- **Logs**: View logs for a specific service using `docker compose logs -f <service_name>`.
+- **Otel Collector**: Check `http://localhost:13133` for health status and `http://localhost:1888/debug/pprof/` for profiling.

.docker/observability/README_ZH.md

@@ -1,27 +1,85 @@
-## 部署可观测性系统
+# RustFS 可观测性技术栈
 
-OpenTelemetry Collector 提供了一个厂商中立的遥测数据处理方案，用于接收、处理和导出遥测数据。它消除了为支持多种开源可观测性数据格式（如
-Jaeger、Prometheus 等）而需要运行和维护多个代理/收集器的必要性。
+本目录包含 RustFS 的全面可观测性技术栈，旨在提供对应用程序性能、日志和追踪的深入洞察。
 
-### 快速部署
+## 组件
 
-1. 进入 `.docker/observability` 目录
-2. 执行以下命令启动服务：
+该技术栈由以下一流的开源组件组成：
+
+- **Prometheus** (v2.53.1): 行业标准的指标收集和告警工具。
+- **Grafana** (v11.1.0): 领先的可观测性可视化平台。
+- **Loki** (v3.1.0): 水平可扩展、高可用、多租户的日志聚合系统。
+- **Tempo** (v2.5.0): 高吞吐量、最小依赖的分布式追踪后端。
+- **Jaeger** (v1.59.0): 分布式追踪系统（配置为辅助 UI/存储）。
+- **OpenTelemetry Collector** (v0.104.0): 接收、处理和导出遥测数据的供应商无关实现。
+
+## 架构
+
+1.  **遥测收集**: 应用程序将 OTLP (OpenTelemetry Protocol) 数据（指标、日志、追踪）发送到 **OpenTelemetry Collector**。
+2.  **处理与导出**: Collector 处理数据（批处理、内存限制）并将其导出到相应的后端：
+    -   **追踪** -> **Tempo** (主要) & **Jaeger** (辅助/可选)
+    -   **指标** -> **Prometheus** (通过抓取 Collector 的导出器)
+    -   **日志** -> **Loki**
+3.  **可视化**: **Grafana** 连接到所有后端（Prometheus, Tempo, Loki, Jaeger），提供统一的仪表盘体验。
+
+## 特性
+
+-   **完全持久化**: 所有数据（指标、日志、追踪）都持久化到 Docker 卷，确保重启后无数据丢失。
+-   **关联性**: 在 Grafana 中实现指标、日志和追踪之间的无缝导航。
+    -   从指标峰值跳转到相关追踪。
+    -   从追踪跳转到相关日志。
+-   **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
+-   **标准化协议**: 完全基于 OpenTelemetry 标准构建。
+
+## 快速开始
+
+### 前置条件
+
+-   Docker
+-   Docker Compose
+
+### 部署
+
+运行以下命令启动整个技术栈：
 
 ```bash
-docker compose -f docker-compose.yml  up -d
+docker compose up -d
 ```
 
-### 访问监控面板
+### 访问仪表盘
 
-服务启动后，可通过以下地址访问各个监控面板：
+| 服务 | URL | 凭据 | 描述 |
+| :--- | :--- | :--- | :--- |
+| **Grafana** | [http://localhost:3000](http://localhost:3000) | `admin` / `admin` | 主要可视化中心。 |
+| **Prometheus** | [http://localhost:9090](http://localhost:9090) | - | 指标查询和状态。 |
+| **Jaeger UI** | [http://localhost:16686](http://localhost:16686) | - | 辅助追踪可视化。 |
+| **Tempo** | [http://localhost:3200](http://localhost:3200) | - | Tempo 状态/指标。 |
 
-- Grafana: `http://localhost:3000` (默认账号/密码：`admin`/`admin`)
-- Jaeger: `http://localhost:16686`
-- Prometheus: `http://localhost:9090`
+## 配置
 
-## 配置可观测性
+### 数据持久化
 
-```shell
-export RUSTFS_OBS_ENDPOINT="http://localhost:4317" # OpenTelemetry Collector 地址
+数据存储在以下 Docker 卷中：
+
+-   `prometheus-data`: Prometheus 指标
+-   `tempo-data`: Tempo 追踪 (WAL 和 Blocks)
+-   `loki-data`: Loki 日志 (Chunks 和 Rules)
+-   `jaeger-data`: Jaeger 追踪 (Badger DB)
+
+要清除所有数据：
+
+```bash
+docker compose down -v
 ```
+
+### 自定义
+
+-   **Prometheus**: 编辑 `prometheus.yml` 以添加抓取目标或告警规则。
+-   **Grafana**: 仪表盘和数据源从 `grafana/` 目录预置。
+-   **Collector**: 编辑 `otel-collector-config.yaml` 以修改管道、处理器或导出器。
+
+## 故障排除
+
+-   **服务健康**: 使用 `docker compose ps` 检查服务健康状况。
+-   **日志**: 使用 `docker compose logs -f <service_name>` 查看特定服务的日志。
+-   **Otel Collector**: 检查 `http://localhost:13133` 获取健康状态，检查 `http://localhost:1888/debug/pprof/` 进行性能分析。

.docker/observability/docker-compose-example-for-rustfs.yml

@@ -0,0 +1,270 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+services:
+  rustfs:
+    security_opt:
+      - "no-new-privileges:true"
+    image: rustfs/rustfs:latest
+    container_name: rustfs-server
+    ports:
+      - "9000:9000" # S3 API port
+      - "9001:9001" # Console port
+    environment:
+      - RUSTFS_VOLUMES=/data/rustfs
+      - RUSTFS_ADDRESS=0.0.0.0:9000
+      - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_CORS_ALLOWED_ORIGINS=*
+      - RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS=*
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
+      - RUSTFS_OBS_LOGGER_LEVEL=info
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - RUSTFS_OBS_PROFILING_ENDPOINT=http://pyroscope:4040
+    volumes:
+      - rustfs-data:/data/rustfs
+    networks:
+      - otel-network
+    restart: unless-stopped
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "sh",
+          "-c",
+          "curl -f http://127.0.0.1:9000/health && curl -f http://127.0.0.1:9001/rustfs/console/health",
+        ]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    depends_on:
+      otel-collector:
+        condition: service_started
+
+  rustfs-init:
+    image: alpine
+    container_name: rustfs-init
+    volumes:
+      - rustfs-data:/data
+    networks:
+      - otel-network
+    command: >
+      sh -c "
+        chown -R 10001:10001 /data &&
+        echo 'Volume Permissions fixed' &&
+        exit 0
+      "
+    restart: no
+
+  # --- Tracing ---
+
+  tempo:
+    image: grafana/tempo:latest
+    container_name: tempo
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml:ro
+      - tempo-data:/var/tempo
+    ports:
+      - "3200:3200" # tempo
+      - "4317" # otlp grpc
+      - "4318" # otlp http
+    networks:
+      - otel-network
+    restart: unless-stopped
+    depends_on:
+      - redpanda
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
+
+  redpanda:
+    image: redpandadata/redpanda:latest # for tempo ingest
+    container_name: redpanda
+    ports:
+      - "9092:9092"
+    networks:
+      - otel-network
+    restart: unless-stopped
+    command: >
+      redpanda start --overprovisioned
+      --mode=dev-container
+      --kafka-addr=PLAINTEXT://0.0.0.0:9092
+      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
+
+  jaeger:
+    image: jaegertracing/jaeger:latest
+    container_name: jaeger
+    environment:
+      - SPAN_STORAGE_TYPE=badger
+      - BADGER_EPHEMERAL=false
+      - BADGER_DIRECTORY_VALUE=/badger/data
+      - BADGER_DIRECTORY_KEY=/badger/key
+      - COLLECTOR_OTLP_ENABLED=true
+    volumes:
+      - ./jaeger.yaml:/etc/jaeger/config.yml
+      - jaeger-data:/badger
+    ports:
+      - "16686:16686" # Web UI
+      - "14269:14269" # Admin/Metrics
+      - "4317" # otlp grpc
+      - "4318" # otlp http
+    command: [ "--config", "/etc/jaeger/config.yml" ]
+    networks:
+      - otel-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:14269" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
+
+  # --- Metrics ---
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    ports:
+      - "9090:9090"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--web.enable-otlp-receiver" # Enable OTLP
+      - "--web.enable-remote-write-receiver" # Enable remote write
+      - "--enable-feature=promql-experimental-functions" # Enable info()
+      - "--storage.tsdb.retention.time=30d"
+    restart: unless-stopped
+    networks:
+      - otel-network
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  # --- Logging ---
+
+  loki:
+    image: grafana/loki:latest
+    container_name: loki
+    volumes:
+      - ./loki.yaml:/etc/loki/loki.yaml:ro
+      - loki-data:/loki
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/loki.yaml
+    networks:
+      - otel-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+
+  # --- Collection ---
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
+    ports:
+      - "1888:1888" # pprof
+      - "8888:8888" # Prometheus metrics for Collector
+      - "8889:8889" # Prometheus metrics for application indicators
+      - "13133:13133" # health check
+      - "4317:4317" # OTLP gRPC
+      - "4318:4318" # OTLP HTTP
+      - "55679:55679" # zpages
+    networks:
+      - otel-network
+    restart: unless-stopped
+    depends_on:
+      - tempo
+      - jaeger
+      - prometheus
+      - loki
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  # --- Profiles ---
+
+  pyroscope:
+    image: grafana/pyroscope:latest
+    container_name: pyroscope
+    ports:
+      - "4040:4040"
+    command:
+      - -self-profiling.disable-push=true
+    networks:
+      - otel-network
+    restart: unless-stopped
+
+  # --- Visualization ---
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_SECURITY_ADMIN_USER=admin
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/etc/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
+    networks:
+      - otel-network
+    restart: unless-stopped
+    depends_on:
+      - prometheus
+      - tempo
+      - loki
+    healthcheck:
+      test:
+        [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+volumes:
+  rustfs-data:
+  tempo-data:
+  jaeger-data:
+  prometheus-data:
+  loki-data:
+  grafana-data:
+
+networks:
+  otel-network:
+    driver: bridge
+    name: "network_otel"
+    ipam:
+      config:
+        - subnet: 172.28.0.0/16
+    driver_opts:
+      com.docker.network.enable_ipv6: "true"

.docker/observability/docker-compose-tempo-ha-override.yml

@@ -0,0 +1,62 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Docker Compose override file for High Availability Tempo setup
+#
+# Usage:
+# docker-compose -f docker-compose-example-for-rustfs.yml \
+#                -f docker-compose-tempo-ha-override.yml up
+
+services:
+  # Override Tempo to use high-availability configuration
+  tempo:
+    volumes:
+      - ./tempo-ha.yaml:/etc/tempo.yaml:ro
+      - tempo-data:/var/tempo
+    ports:
+      - "3200:3200"   # Tempo HTTP
+      - "4317:4317"   # OTLP gRPC
+      - "4318:4318"   # OTLP HTTP
+      - "7946:7946"   # Memberlist
+      - "14250:14250" # Jaeger gRPC
+      - "14268:14268" # Jaeger Thrift HTTP
+      - "9411:9411"   # Zipkin
+    environment:
+      - TEMPO_MEMBERLIST_BIND_PORT=7946
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    depends_on:
+      - redpanda
+
+volumes:
+  tempo-data:
+    driver: local
+    driver_opts:
+      type: tmpfs
+      device: tmpfs
+      o: "size=4g"  # Allocate 4GB tmpfs for Tempo data (adjust based on your needs)
+
+# Network configuration remains the same
+# networks:
+#   otel-network:
+#     driver: bridge
+#     name: "network_otel"
+#     ipam:
+#       config:
+#         - subnet: 172.28.0.0/16
+

.docker/observability/docker-compose.yml

@@ -14,41 +14,108 @@
 
 services:
 
-  tempo-init:
-    image: busybox:latest
-    command: [ "sh", "-c", "chown -R 10001:10001 /var/tempo" ]
-    volumes:
-      - ./tempo-data:/var/tempo
-    user: root
-    networks:
-      - otel-network
-    restart: "no"
+  # --- Tracing ---
 
   tempo:
     image: grafana/tempo:latest
-    user: "10001" # The container must be started with root to execute chown in the script
-    command: [ "-config.file=/etc/tempo.yaml" ] # This is passed as a parameter to the entry point script
+    container_name: tempo
+    command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
-      - ./tempo-data:/var/tempo
+      - tempo-data:/var/tempo
     ports:
       - "3200:3200" # tempo
-      - "24317:4317" # otlp grpc
-      - "24318:4318" # otlp http
-    restart: unless-stopped
+      - "4317" # otlp grpc
+      - "4318" # otlp http
+      - "7946" # memberlist
     networks:
       - otel-network
+    restart: unless-stopped
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/metrics" ]
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 15s
 
+  jaeger:
+    image: jaegertracing/jaeger:latest
+    container_name: jaeger
+    environment:
+      - SPAN_STORAGE_TYPE=badger
+      - BADGER_EPHEMERAL=false
+      - BADGER_DIRECTORY_VALUE=/badger/data
+      - BADGER_DIRECTORY_KEY=/badger/key
+      - COLLECTOR_OTLP_ENABLED=true
+    volumes:
+      - ./jaeger.yaml:/etc/jaeger/config.yml
+      - jaeger-data:/badger
+    ports:
+      - "16686:16686" # Web UI
+      - "14269:14269" # Admin/Metrics
+      - "4317" # otlp grpc
+      - "4318" # otlp http
+    command: [ "--config", "/etc/jaeger/config.yml" ]
+    networks:
+      - otel-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:14269" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
+
+  # --- Metrics ---
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    ports:
+      - "9090:9090"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--web.enable-otlp-receiver" # Enable OTLP
+      - "--web.enable-remote-write-receiver" # Enable remote write
+      - "--enable-feature=promql-experimental-functions" # Enable info()
+      - "--storage.tsdb.retention.time=30d"
+    restart: unless-stopped
+    networks:
+      - otel-network
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  # --- Logging ---
+
+  loki:
+    image: grafana/loki:latest
+    container_name: loki
+    volumes:
+      - ./loki.yaml:/etc/loki/loki.yaml:ro
+      - loki-data:/loki
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/loki.yaml
+    networks:
+      - otel-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+
+  # --- Collection ---
+
   otel-collector:
     image: otel/opentelemetry-collector-contrib:latest
-    environment:
-      - TZ=Asia/Shanghai
     volumes:
       - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
     ports:
@@ -61,116 +128,70 @@ services:
       - "55679:55679" # zpages
     networks:
       - otel-network
+    restart: unless-stopped
     depends_on:
-      jaeger:
-        condition: service_started
-      tempo:
-        condition: service_started
-      prometheus:
-        condition: service_started
-      loki:
-        condition: service_started
+      - tempo
+      - jaeger
+      - prometheus
+      - loki
     healthcheck:
       test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
       interval: 10s
       timeout: 5s
       retries: 3
 
-  jaeger:
-    image: jaegertracing/jaeger:latest
-    environment:
-      - TZ=Asia/Shanghai
-      - SPAN_STORAGE_TYPE=memory
-      - COLLECTOR_OTLP_ENABLED=true
+  # --- Profiles ---
+
+  pyroscope:
+    image: grafana/pyroscope:latest
+    container_name: pyroscope
     ports:
-      - "16686:16686" # Web UI
-      - "14317:4317" # OTLP gRPC
-      - "14318:4318" # OTLP HTTP
-      - "18888:8888" # collector
-    networks:
-      - otel-network
-    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:16686" ]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-  prometheus:
-    image: prom/prometheus:latest
-    environment:
-      - TZ=Asia/Shanghai
-    volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - ./prometheus-data:/prometheus
-    ports:
-      - "9090:9090"
+      - "4040:4040"
     command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--web.enable-otlp-receiver' # Enable OTLP
-      - '--web.enable-remote-write-receiver' # Enable remote write
-      - '--enable-feature=promql-experimental-functions' # Enable info()
-      - '--storage.tsdb.min-block-duration=15m' # Minimum block duration
-      - '--storage.tsdb.max-block-duration=1h'  # Maximum block duration
-      - '--log.level=info'
-      - '--storage.tsdb.retention.time=30d'
-      - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
+      - -self-profiling.disable-push=true
+    networks:
+      - otel-network
     restart: unless-stopped
-    networks:
-      - otel-network
-    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-  loki:
-    image: grafana/loki:latest
-    environment:
-      - TZ=Asia/Shanghai
-    volumes:
-      - ./loki-config.yaml:/etc/loki/local-config.yaml:ro
-    ports:
-      - "3100:3100"
-    command: -config.file=/etc/loki/local-config.yaml
-    networks:
-      - otel-network
-    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
-      interval: 10s
-      timeout: 5s
-      retries: 3
+
+  # --- Visualization ---
+
   grafana:
     image: grafana/grafana:latest
+    container_name: grafana
     ports:
-      - "3000:3000"  # Web UI
-    volumes:
-      - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+      - "3000:3000"
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_SECURITY_ADMIN_USER=admin
-      - TZ=Asia/Shanghai
-      - GF_INSTALL_PLUGINS=grafana-pyroscope-datasource
-    restart: unless-stopped
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/etc/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
     networks:
       - otel-network
+    restart: unless-stopped
     depends_on:
       - prometheus
       - tempo
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
+      test:
+        [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
       interval: 10s
       timeout: 5s
       retries: 3
 
 volumes:
-  prometheus-data:
   tempo-data:
+  jaeger-data:
+  prometheus-data:
+  loki-data:
+  grafana-data:
 
 networks:
   otel-network:
     driver: bridge
-    name: "network_otel_config"
+    name: "network_otel"
     ipam:
       config:
         - subnet: 172.28.0.0/16

.docker/observability/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,25 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: 1
+
+providers:
+  - name: "default"
+    orgId: 1
+    folder: ""
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    options:
+      path: /etc/grafana/dashboards

.docker/observability/grafana/provisioning/datasources.yaml

@@ -1,3 +1,17 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 apiVersion: 1
 
 datasources:
@@ -7,102 +21,77 @@ datasources:
     access: proxy
     orgId: 1
     url: http://prometheus:9090
-    basicAuth: false
-    isDefault: false
+    isDefault: true
     version: 1
     editable: false
     jsonData:
       httpMethod: GET
+      exemplarTraceIdDestinations:
+        - name: trace_id
+          datasourceUid: tempo
+
   - name: Tempo
     type: tempo
+    uid: tempo
     access: proxy
     orgId: 1
     url: http://tempo:3200
-    basicAuth: false
-    isDefault: true
+    isDefault: false
     version: 1
     editable: false
-    apiVersion: 1
-    uid: tempo
     jsonData:
       httpMethod: GET
       serviceMap:
         datasourceUid: prometheus
-      streamingEnabled:
-        search: true
-      tracesToLogsV2:
-        # Field with an internal link pointing to a logs data source in Grafana.
-        # datasourceUid value must match the uid value of the logs data source.
-        datasourceUid: 'loki'
-        spanStartTimeShift: '-1h'
-        spanEndTimeShift: '1h'
-        tags: [ 'job', 'instance', 'pod', 'namespace' ]
-        filterByTraceID: false
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '1s'
+        spanEndTimeShift: '-1s'
+        filterByTraceID: true
         filterBySpanID: false
-        customQuery: true
-        query: 'method="$${__span.tags.method}"'
-        tracesToMetrics:
-          datasourceUid: 'prometheus'
-          spanStartTimeShift: '-1h'
-          spanEndTimeShift: '1h'
-          tags: [ { key: 'service.name', value: 'service' }, { key: 'job' } ]
-          queries:
-            - name: 'Sample query'
-              query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
-        tracesToProfiles:
-          datasourceUid: 'grafana-pyroscope-datasource'
-          tags: [ 'job', 'instance', 'pod', 'namespace' ]
-          profileTypeId: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds'
-          customQuery: true
-          query: 'method="$${__span.tags.method}"'
-        serviceMap:
-          datasourceUid: 'prometheus'
-        nodeGraph:
-          enabled: true
-        search:
-          hide: false
-        traceQuery:
-          timeShiftEnabled: true
-          spanStartTimeShift: '-1h'
-          spanEndTimeShift: '1h'
-        spanBar:
-          type: 'Tag'
-          tag: 'http.path'
-        streamingEnabled:
-          search: true
-  - name: Jaeger
-    type: jaeger
-    uid: Jaeger
-    url: http://jaeger:16686
-    basicAuth: false
-    access: proxy
-    readOnly: false
-    isDefault: false
-    jsonData:
-      tracesToLogsV2:
-        # Field with an internal link pointing to a logs data source in Grafana.
-        # datasourceUid value must match the uid value of the logs data source.
-        datasourceUid: 'loki'
-        spanStartTimeShift: '1h'
-        spanEndTimeShift: '-1h'
-        tags: [ 'job', 'instance', 'pod', 'namespace' ]
-        filterByTraceID: false
-        filterBySpanID: false
-        customQuery: true
-        query: 'method="$${__span.tags.method}"'
       tracesToMetrics:
-        datasourceUid: 'Prometheus'
-        spanStartTimeShift: '1h'
-        spanEndTimeShift: '-1h'
-        tags: [ { key: 'service.name', value: 'service' }, { key: 'job' } ]
+        datasourceUid: prometheus
+        tags: [ { key: 'service.name' }, { key: 'job' } ]
         queries:
-          - name: 'Sample query'
-            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
+          - name: 'Service-Level Latency'
+            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
+          - name: 'Service-Level Calls'
+            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
+          - name: 'Service-Level Errors'
+            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
       nodeGraph:
         enabled: true
-      traceQuery:
-        timeShiftEnabled: true
-        spanStartTimeShift: '1h'
-        spanEndTimeShift: '-1h'
-      spanBar:
-        type: 'None'
+
+  - name: Loki
+    type: loki
+    uid: loki
+    orgId: 1
+    url: http://loki:3100
+    isDefault: false
+    version: 1
+    editable: false
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: 'trace_id=(\w+)'
+          name: 'TraceID'
+          url: '$${__value.raw}'
+
+  - name: Jaeger
+    type: jaeger
+    uid: jaeger
+    url: http://jaeger:16686
+    access: proxy
+    isDefault: false
+    editable: false
+    jsonData:
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '1s'
+        spanEndTimeShift: '-1s'
+        filterByTraceID: true
+        filterBySpanID: false

.docker/observability/grafana/provisioning/datasources/datasources.yaml

@@ -0,0 +1,98 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    url: http://prometheus:9090
+    access: proxy
+    isDefault: true
+    editable: false
+    jsonData:
+      httpMethod: GET
+      exemplarTraceIdDestinations:
+        - name: trace_id
+          datasourceUid: tempo
+
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    access: proxy
+    url: http://tempo:3200
+    isDefault: false
+    editable: false
+    jsonData:
+      httpMethod: GET
+      serviceMap:
+        datasourceUid: prometheus
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '-1h'
+        spanEndTimeShift: '1h'
+        filterByTraceID: true
+        filterBySpanID: false
+      tracesToMetrics:
+        datasourceUid: prometheus
+        tags: [ { key: 'service.name' }, { key: 'job' } ]
+        queries:
+          - name: 'Service-Level Latency'
+            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
+          - name: 'Service-Level Calls'
+            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
+          - name: 'Service-Level Errors'
+            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
+      nodeGraph:
+        enabled: true
+
+  - name: Loki
+    type: loki
+    uid: loki
+    url: http://loki:3100
+    basicAuth: false
+    isDefault: false
+    editable: false
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: 'trace_id=(\w+)'
+          name: 'TraceID'
+          url: '$${__value.raw}'
+
+  - name: Jaeger
+    type: jaeger
+    uid: jaeger
+    url: http://jaeger:16686
+    access: proxy
+    isDefault: false
+    editable: false
+    jsonData:
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '1s'
+        spanEndTimeShift: '-1s'
+        filterByTraceID: true
+        filterBySpanID: false
+
+  - name: Pyroscope
+    type: grafana-pyroscope-datasource
+    url: http://pyroscope:4040
+    jsonData:
+      minStep: '15s'

.docker/observability/jaeger-config.yaml

@@ -1,114 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-service:
-  extensions: [ jaeger_storage, jaeger_query, remote_sampling, healthcheckv2 ]
-  pipelines:
-    traces:
-      receivers: [ otlp, jaeger, zipkin ]
-      processors: [ batch, adaptive_sampling ]
-      exporters: [ jaeger_storage_exporter ]
-  telemetry:
-    resource:
-      service.name: jaeger
-    metrics:
-      level: detailed
-      readers:
-        - pull:
-            exporter:
-              prometheus:
-                host: 0.0.0.0
-                port: 8888
-    logs:
-      level: debug
-    # TODO Initialize telemetry tracer once OTEL released new feature.
-    # https://github.com/open-telemetry/opentelemetry-collector/issues/10663
-
-extensions:
-  healthcheckv2:
-    use_v2: true
-    http:
-
-  # pprof:
-  #   endpoint: 0.0.0.0:1777
-  # zpages:
-  #   endpoint: 0.0.0.0:55679
-
-  jaeger_query:
-    storage:
-      traces: some_store
-      traces_archive: another_store
-    ui:
-      config_file: ./cmd/jaeger/config-ui.json
-      log_access: true
-    # The maximum duration that is considered for clock skew adjustments.
-    # Defaults to 0 seconds, which means it's disabled.
-    max_clock_skew_adjust: 0s
-    grpc:
-      endpoint: 0.0.0.0:16685
-    http:
-      endpoint: 0.0.0.0:16686
-
-  jaeger_storage:
-    backends:
-      some_store:
-        memory:
-          max_traces: 1000000
-          max_events: 100000
-      another_store:
-        memory:
-          max_traces: 1000000
-    metric_backends:
-      some_metrics_storage:
-        prometheus:
-          endpoint: http://prometheus:9090
-          normalize_calls: true
-          normalize_duration: true
-
-  remote_sampling:
-    # You can either use file or adaptive sampling strategy in remote_sampling
-    # file:
-    #   path: ./cmd/jaeger/sampling-strategies.json
-    adaptive:
-      sampling_store: some_store
-      initial_sampling_probability: 0.1
-    http:
-    grpc:
-
-receivers:
-  otlp:
-    protocols:
-      grpc:
-      http:
-
-  jaeger:
-    protocols:
-      grpc:
-      thrift_binary:
-      thrift_compact:
-      thrift_http:
-
-  zipkin:
-
-processors:
-  batch:
-  metadata_keys: [ "span.kind", "http.method", "http.status_code", "db.system", "db.statement", "messaging.system", "messaging.destination", "messaging.operation","span.events","span.links" ]
-  # Adaptive Sampling Processor is required to support adaptive sampling.
-  # It expects remote_sampling extension with `adaptive:` config to be enabled.
-  adaptive_sampling:
-
-exporters:
-  jaeger_storage_exporter:
-    trace_storage: some_store
-

.docker/observability/jaeger.yaml

@@ -0,0 +1,74 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+service:
+  extensions: [jaeger_storage, jaeger_query]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [jaeger_storage_exporter, spanmetrics]
+    metrics/spanmetrics:
+      receivers: [spanmetrics]
+      exporters: [prometheus]
+  telemetry:
+    resource:
+      service.name: jaeger
+    metrics:
+      level: detailed
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: 0.0.0.0
+                port: 8888
+    logs:
+      level: DEBUG
+
+extensions:
+  jaeger_query:
+    storage:
+      traces: some_storage
+      metrics: some_metrics_storage
+  jaeger_storage:
+    backends:
+      some_storage:
+        memory:
+          max_traces: 100000
+    metric_backends:
+      some_metrics_storage:
+        prometheus:
+          endpoint: http://prometheus:9090
+          normalize_calls: true
+          normalize_duration: true
+
+connectors:
+  spanmetrics:
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "0.0.0.0:4317"
+      http:
+        endpoint: "0.0.0.0:4318"
+
+processors:
+  batch:
+
+exporters:
+  jaeger_storage_exporter:
+    trace_storage: some_storage
+  prometheus:
+    endpoint: "0.0.0.0:8889"

.docker/observability/loki.yaml

@@ -11,22 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 auth_enabled: false
 
 server:
   http_listen_port: 3100
-  grpc_listen_port: 9096
-  log_level: debug
+  grpc_listen_port: 9095
+  log_level: info
   grpc_server_max_concurrent_streams: 1000
 
 common:
   instance_addr: 127.0.0.1
-  path_prefix: /tmp/loki
+  path_prefix: /loki
   storage:
     filesystem:
-      chunks_directory: /tmp/loki/chunks
-      rules_directory: /tmp/loki/rules
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
   replication_factor: 1
   ring:
     kvstore:
@@ -39,12 +38,6 @@ query_range:
         enabled: true
         max_size_mb: 100
 
-limits_config:
-  metric_aggregation_enabled: true
-  max_line_size: 256KB
-  max_line_size_truncate: false
-  allow_structured_metadata: true
-
 schema_config:
   configs:
     - from: 2020-10-24
@@ -54,29 +47,17 @@ schema_config:
       index:
         prefix: index_
         period: 24h
-      row_shards: 16
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  allow_structured_metadata: true
+  max_line_size: 256KB
 
 pattern_ingester:
   enabled: true
   metric_aggregation:
     loki_address: localhost:3100
 
-ruler:
-  alertmanager_url: http://localhost:9093
-
 frontend:
   encoding: protobuf
-
-
-# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
-# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
-#
-# Statistics help us better understand how Loki is used, and they show us performance
-# levels for most users. This helps us prioritize features and documentation.
-# For more information on what's sent, look at
-# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
-# Refer to the buildReport method to see what goes into a report.
-#
-# If you would like to disable reporting, uncomment the following lines:
-#analytics:
-#  reporting_enabled: false

.docker/observability/otel-collector-config.yaml

@@ -15,69 +15,70 @@
 receivers:
   otlp:
     protocols:
-      grpc: # OTLP gRPC receiver
+      grpc:
         endpoint: 0.0.0.0:4317
-      http: # OTLP HTTP receiver
+      http:
         endpoint: 0.0.0.0:4318
 
 processors:
-  batch: # Batch processor to improve throughput
-    timeout: 5s
-    send_batch_size: 1000
-    metadata_keys: [ ]
-    metadata_cardinality_limit: 1000
+  batch:
+    timeout: 1s
+    send_batch_size: 1024
   memory_limiter:
     check_interval: 1s
-    limit_mib: 512
+    limit_mib: 1024
+    spike_limit_mib: 256
   transform/logs:
     log_statements:
       - context: log
         statements:
-          # Extract Body as attribute "message"
           - set(attributes["message"], body.string)
-          # Retain the original Body
           - set(attributes["log.body"], body.string)
 
 exporters:
-  otlp/traces: # OTLP exporter for trace data
-    endpoint: "http://jaeger:4317"  # OTLP gRPC endpoint for Jaeger
+  otlp/tempo:
+    endpoint: "tempo:4317"
     tls:
-      insecure: true  # TLS is disabled in the development environment and a certificate needs to be configured in the production environment.
-    compression: gzip  # Enable compression to reduce network bandwidth
+      insecure: true
+    compression: gzip
     retry_on_failure:
-      enabled: true  # Enable retry on failure
-      initial_interval: 1s  # Initial interval for retry
-      max_interval: 30s  # Maximum interval for retry
-      max_elapsed_time: 300s  # Maximum elapsed time for retry
+      enabled: true
+      initial_interval: 1s
+      max_interval: 30s
+      max_elapsed_time: 300s
     sending_queue:
-      enabled: true  # Enable sending queue
-      num_consumers: 10  # Number of consumers
-      queue_size: 5000  # Queue size
-  otlp/tempo: # OTLP exporter for trace data
-    endpoint: "http://tempo:4317"  # OTLP gRPC endpoint for tempo
+      enabled: true
+      num_consumers: 10
+      queue_size: 5000
+
+  otlp/jaeger:
+    endpoint: "jaeger:4317"
     tls:
-      insecure: true  # TLS is disabled in the development environment and a certificate needs to be configured in the production environment.
-    compression: gzip  # Enable compression to reduce network bandwidth
+      insecure: true
+    compression: gzip
     retry_on_failure:
-      enabled: true  # Enable retry on failure
-      initial_interval: 1s  # Initial interval for retry
-      max_interval: 30s  # Maximum interval for retry
-      max_elapsed_time: 300s  # Maximum elapsed time for retry
+      enabled: true
+      initial_interval: 1s
+      max_interval: 30s
+      max_elapsed_time: 300s
     sending_queue:
-      enabled: true  # Enable sending queue
-      num_consumers: 10  # Number of consumers
-      queue_size: 5000  # Queue size
-  prometheus: # Prometheus exporter for metrics data
-    endpoint: "0.0.0.0:8889"  # Prometheus scraping endpoint
-    send_timestamps: true  # Send timestamp
-    metric_expiration: 5m  # Metric expiration time
+      enabled: true
+      num_consumers: 10
+      queue_size: 5000
+
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+    send_timestamps: true
+    metric_expiration: 5m
     resource_to_telemetry_conversion:
-      enabled: true  # Enable resource to telemetry conversion
-  otlphttp/loki: # Loki exporter for log data
+      enabled: true
+
+  otlphttp/loki:
     endpoint: "http://loki:3100/otlp"
     tls:
       insecure: true
-    compression: gzip  # Enable compression to reduce network bandwidth
+    compression: gzip
+
 extensions:
   health_check:
     endpoint: 0.0.0.0:13133
@@ -85,13 +86,14 @@ extensions:
     endpoint: 0.0.0.0:1888
   zpages:
     endpoint: 0.0.0.0:55679
+
 service:
-  extensions: [ health_check, pprof, zpages ]  # Enable extension
+  extensions: [ health_check, pprof, zpages ]
   pipelines:
     traces:
       receivers: [ otlp ]
       processors: [ memory_limiter, batch ]
-      exporters: [ otlp/traces, otlp/tempo ]
+      exporters: [ otlp/tempo, otlp/jaeger ]
     metrics:
       receivers: [ otlp ]
       processors: [ batch ]
@@ -102,20 +104,13 @@ service:
       exporters: [ otlphttp/loki ]
   telemetry:
     logs:
-      level: "debug"  # Collector log level
-      encoding: "json"  # Log encoding: console or json
+      level: "info"
+      encoding: "json"
     metrics:
-      level: "detailed" # Can be basic, normal, detailed
+      level: "normal"
       readers:
-        - periodic:
-            exporter:
-              otlp:
-                protocol: http/protobuf
-                endpoint: http://otel-collector:4318
         - pull:
             exporter:
               prometheus:
                 host: '0.0.0.0'
                 port: 8888
-
-

.docker/observability/prometheus-data/.gitignore

@@ -1 +0,0 @@
-*

.docker/observability/prometheus.yml

@@ -17,27 +17,40 @@ global:
   evaluation_interval: 15s
   external_labels:
     cluster: 'rustfs-dev'  # Label to identify the cluster
-    relica: '1'  # Replica identifier
+    replica: '1'  # Replica identifier
 
 scrape_configs:
-  - job_name: 'otel-collector-internal'
+  - job_name: 'otel-collector'
     static_configs:
       - targets: [ 'otel-collector:8888' ]  # Scrape metrics from Collector
     scrape_interval: 10s
+
   - job_name: 'rustfs-app-metrics'
     static_configs:
       - targets: [ 'otel-collector:8889' ]  # Application indicators
     scrape_interval: 15s
     metric_relabel_configs:
+      - source_labels: [ __name__ ]
+        regex: 'go_.*'
+        action: drop  # Drop Go runtime metrics if not needed
+
   - job_name: 'tempo'
     static_configs:
       - targets: [ 'tempo:3200' ]  # Scrape metrics from Tempo
+
   - job_name: 'jaeger'
     static_configs:
-      - targets: [ 'jaeger:8888' ]  # Jaeger admin port
+      - targets: [ 'jaeger:14269' ]  # Jaeger admin port (14269 is standard for admin/metrics)
+
+  - job_name: 'loki'
+    static_configs:
+      - targets: [ 'loki:3100' ]
+
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: [ 'localhost:9090' ]
 
 otlp:
-  # Recommended attributes to be promoted to labels.
   promote_resource_attributes:
     - service.instance.id
     - service.name
@@ -56,10 +69,8 @@ otlp:
     - k8s.pod.name
     - k8s.replicaset.name
     - k8s.statefulset.name
-  # Ingest OTLP data keeping all characters in metric/label names.
   translation_strategy: NoUTF8EscapingWithSuffixes
 
 storage:
-  # OTLP is a push-based protocol, Out of order samples is a common scenario.
   tsdb:
-    out_of_order_time_window: 30m
+    out_of_order_time_window: 30m

.docker/observability/tempo-data/.gitignore

@@ -1 +0,0 @@
-*

.docker/observability/tempo-ha.yaml

@@ -0,0 +1,286 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# High Availability Tempo Configuration for docker-compose-example-for-rustfs.yml
+# Features:
+# - Distributed architecture with multiple components
+# - Kafka-based ingestion for fault tolerance
+# - Replication factor of 3 for data resilience
+# - Query frontend for load balancing
+# - Metrics generation from traces
+# - WAL for durability
+
+partition_ring_live_store: true
+stream_over_http_enabled: true
+
+server:
+  http_listen_port: 3200
+  http_server_read_timeout: 30s
+  http_server_write_timeout: 30s
+  grpc_server_max_recv_msg_size: 4194304  # 4MB
+  grpc_server_max_send_msg_size: 4194304
+  log_level: info
+  log_format: json
+
+# Memberlist configuration for distributed mode
+memberlist:
+  node_name: tempo
+  bind_port: 7946
+  join_members:
+    - tempo:7946
+  retransmit_factor: 4
+  node_timeout: 15s
+  retransmit_interval: 300ms
+  dead_node_reclaim_time: 30s
+
+# Distributor configuration - receives traces and routes to ingesters
+distributor:
+  ingester_write_path_enabled: true
+  kafka_write_path_enabled: true
+  rate_limit_bytes: 10MB
+  rate_limit_enabled: true
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: "0.0.0.0:4317"
+          max_concurrent_streams: 0
+          max_receive_message_size: 4194304
+        http:
+          endpoint: "0.0.0.0:4318"
+          cors:
+            allowed_origins:
+              - "*"
+            max_age: 86400
+    jaeger:
+      protocols:
+        grpc:
+          endpoint: "0.0.0.0:14250"
+        thrift_http:
+          endpoint: "0.0.0.0:14268"
+    zipkin:
+      endpoint: "0.0.0.0:9411"
+  ring:
+    kvstore:
+      store: memberlist
+    heartbeat_timeout: 5s
+    replication_factor: 3
+    heartbeat_interval: 5s
+
+# Ingester configuration - stores traces and querying
+ingester:
+  lifecycler:
+    address: tempo
+    ring:
+      kvstore:
+        store: memberlist
+      replication_factor: 3
+      max_cache_freshness_per_sec: 10s
+      heartbeat_interval: 5s
+      heartbeat_timeout: 5s
+    num_tokens: 128
+    tokens_file_path: /var/tempo/tokens.json
+    claim_on_rollout: true
+  trace_idle_period: 20s
+  max_block_bytes: 10_000_000
+  max_block_duration: 10m
+  chunk_size_bytes: 1_000_000
+  chunk_encoding: snappy
+  wal:
+    checkpoint_duration: 5s
+    max_wal_blocks: 4
+  metrics:
+    enabled: true
+    level: block
+    target_info_duration: 15m
+
+# WAL configuration for data durability
+wal:
+  checkpoint_duration: 5s
+  flush_on_shutdown: true
+  path: /var/tempo/wal
+
+# Kafka ingestion configuration - for high throughput scenarios
+ingest:
+  enabled: true
+  kafka:
+    brokers: [ redpanda:9092 ]
+    topic: tempo-ingest
+    encoding: protobuf
+    consumer_group: tempo-ingest-consumer
+    session_timeout: 10s
+    rebalance_timeout: 1m
+    partition: auto
+    verbosity: 2
+
+# Query frontend configuration - distributed querying
+query_frontend:
+  compression: gzip
+  downstream_url: http://localhost:3200
+  log_queries_longer_than: 5s
+  cache_uncompressed_bytes: 100MB
+  max_outstanding_requests_per_tenant: 100
+  max_query_length: 48h
+  max_query_lookback: 30d
+  default_result_cache_ttl: 1m
+  result_cache:
+    cache:
+      enable_fifocache: true
+      default_validity: 1m
+  rf1_after: "1999-01-01T00:00:00Z"
+  mcp_server:
+    enabled: true
+
+# Querier configuration - queries traces
+querier:
+  frontend_worker:
+    frontend_address: localhost:3200
+    grpc_client_config:
+      max_recv_msg_size: 104857600
+  max_concurrent_queries: 20
+  max_metric_bytes_per_trace: 1MB
+
+# Query scheduler configuration - for distributed querying
+query_scheduler:
+  use_scheduler_ring: false
+
+# Metrics generator configuration - generates metrics from traces
+metrics_generator:
+  enabled: true
+  registry:
+    enabled: true
+    external_labels:
+      source: tempo
+      cluster: rustfs-docker-ha
+      environment: production
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+        resource_to_telemetry_conversion:
+          enabled: true
+  processor:
+    batch:
+      timeout: 10s
+      send_batch_size: 1024
+    memory_limiter:
+      check_interval: 5s
+      limit_mib: 512
+      spike_limit_mib: 128
+  processors:
+    - span-metrics
+    - local-blocks
+    - service-graphs
+  generate_native_histograms: both
+
+# Backend worker configuration
+backend_worker:
+  backend_scheduler_addr: localhost:3200
+  compaction:
+    block_retention: 24h
+    compacted_block_retention: 1h
+  ring:
+    kvstore:
+      store: memberlist
+
+# Backend scheduler configuration
+backend_scheduler:
+  enabled: true
+  provider:
+    compaction:
+      compaction:
+        block_retention: 24h
+        compacted_block_retention: 1h
+        concurrency: 25
+        v2_out_path: /var/tempo/blocks/compaction
+
+# Storage configuration - local backend with proper retention
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /var/tempo/wal
+      checkpoint_duration: 5s
+      flush_on_shutdown: true
+    local:
+      path: /var/tempo/blocks
+      bloom_filter_false_positive: 0.05
+      bloom_shift: 4
+      index:
+        downsample_bytes: 1000000
+        page_size_bytes: 0
+        cache_size_bytes: 0
+    pool:
+      max_workers: 400
+      queue_depth: 10000
+
+# Compactor configuration - manages block compaction
+compactor:
+  compaction:
+    block_retention: 168h  # 7 days
+    compacted_block_retention: 1h
+    concurrency: 25
+    v2_out_path: /var/tempo/blocks/compaction
+    shard_count: 32
+    max_block_bytes: 107374182400  # 100GB
+    max_compaction_objects: 6000000
+    max_time_per_tenant: 5m
+    block_size_bytes: 107374182400
+  ring:
+    kvstore:
+      store: memberlist
+    heartbeat_interval: 5s
+    heartbeat_timeout: 5s
+
+# Limits configuration - rate limiting and quotas
+limits:
+  max_traces_per_user: 10000
+  max_bytes_per_trace: 10485760  # 10MB
+  max_search_bytes_per_trace: 0
+  forgiving_oversize_traces: true
+  rate_limit_bytes: 10MB
+  rate_limit_enabled: true
+  ingestion_burst_size_bytes: 20MB
+  ingestion_rate_limit_bytes: 10MB
+  max_bytes_per_second: 10485760
+  metrics_generator_max_active_series: 10000
+  metrics_generator_max_churned_series: 10000
+  metrics_generator_forta_out_of_order_ttl: 5m
+
+# Override configuration
+overrides:
+  defaults:
+    metrics_generator:
+      processors:
+        - span-metrics
+        - local-blocks
+        - service-graphs
+      generate_native_histograms: both
+      max_active_series: 10000
+      max_churned_series: 10000
+
+# Usage reporting configuration
+usage_report:
+  reporting_enabled: false
+
+# Tracing configuration for debugging
+tracing:
+  enabled: true
+  jaeger:
+    sampler:
+      name: probabilistic
+      param: 0.1
+    reporter_log_spans: false
+

.docker/observability/tempo.yaml

@@ -1,33 +1,74 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+partition_ring_live_store: true
 stream_over_http_enabled: true
+
 server:
   http_listen_port: 3200
   log_level: info
 
-query_frontend:
-  search:
-    duration_slo: 5s
-    throughput_bytes_slo: 1.073741824e+09
-    metadata_slo:
-      duration_slo: 5s
-      throughput_bytes_slo: 1.073741824e+09
-  trace_by_id:
-    duration_slo: 5s
+memberlist:
+  node_name: tempo
+  bind_port: 7946
+  join_members:
+    - tempo:7946
 
+# Distributor configuration - receives traces and writes directly to ingesters
 distributor:
+  ingester_write_path_enabled: true
+  kafka_write_path_enabled: false
   receivers:
     otlp:
       protocols:
         grpc:
-          endpoint: "0.0.0.0:4317"
+          endpoint: "tempo:4317"
         http:
-          endpoint: "0.0.0.0:4318"
+          endpoint: "tempo:4318"
+  ring:
+    kvstore:
+      store: memberlist
 
+# Ingester configuration - consumes from Kafka and stores traces
 ingester:
-  max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+  lifecycler:
+    ring:
+      kvstore:
+        store: memberlist
+      replication_factor: 1
+    tokens_file_path: /var/tempo/tokens.json
+  trace_idle_period: 10s
+  max_block_bytes: 1_000_000
+  max_block_duration: 5m
 
-compactor:
+backend_scheduler:
+  provider:
+    compaction:
+      compaction:
+        block_retention: 1h
+
+backend_worker:
+  backend_scheduler_addr: localhost:3200
   compaction:
-    block_retention: 1h # overall Tempo trace retention. set for demo purposes
+    block_retention: 1h
+  ring:
+    kvstore:
+      store: memberlist
+
+querier:
+  frontend_worker:
+    frontend_address: tempo:3200
 
 metrics_generator:
   registry:
@@ -39,19 +80,45 @@ metrics_generator:
     remote_write:
       - url: http://prometheus:9090/api/v1/write
         send_exemplars: true
-  traces_storage:
-    path: /var/tempo/generator/traces
+
+query_frontend:
+  rf1_after: "1999-01-01T00:00:00Z"
+  mcp_server:
+    enabled: true
 
 storage:
   trace:
-    backend: local # backend configuration to use
+    backend: local
     wal:
-      path: /var/tempo/wal # where to store the wal locally
+      path: /var/tempo/wal
     local:
       path: /var/tempo/blocks
 
 overrides:
   defaults:
     metrics_generator:
-      processors: [ service-graphs, span-metrics, local-blocks ] # enables metrics generator
-      generate_native_histograms: both
+      processors: [ "span-metrics", "service-graphs", "local-blocks" ]
+      generate_native_histograms: both
+
+ingest:
+  enabled: false
+  # Disabled because using direct ingester write path
+  # If you want Kafka path, enable this and set:
+  # kafka:
+  #   brokers: [redpanda:9092]
+  #   topic: tempo-ingest
+  #   encoding: protobuf
+  #   consumer_group: tempo-ingest-consumer
+
+block_builder:
+  consume_cycle_duration: 30s
+
+compactor:
+  compaction:
+    block_retention: 168h  # 7 days
+  ring:
+    kvstore:
+      store: memberlist
+
+usage_report:
+  reporting_enabled: false

.docker/openobserve-otel/README.md

@@ -5,71 +5,57 @@
 
 English | [中文](README_ZH.md)
 
-This directory contains the configuration files for setting up an observability stack with OpenObserve and OpenTelemetry
-Collector.
+This directory contains the configuration for an **alternative** observability stack using OpenObserve.
 
-### Overview
+## ⚠️ Note
 
-This setup provides a complete observability solution for your applications:
+For the **recommended** observability stack (Prometheus, Grafana, Tempo, Loki), please see `../observability/`.
 
-- **OpenObserve**: A modern, open-source observability platform for logs, metrics, and traces.
-- **OpenTelemetry Collector**: Collects and processes telemetry data before sending it to OpenObserve.
+## 🌟 Overview
 
-### Setup Instructions
+OpenObserve is a lightweight, all-in-one observability platform that handles logs, metrics, and traces in a single binary. This setup is ideal for:
+- Resource-constrained environments.
+- Quick setup and testing.
+- Users who prefer a unified UI.
 
-1. **Prerequisites**:
-    - Docker and Docker Compose installed
-    - Sufficient memory resources (minimum 2GB recommended)
+## 🚀 Quick Start
 
-2. **Starting the Services**:
-   ```bash
-   cd .docker/openobserve-otel
-   docker compose -f docker-compose.yml  up -d
-   ```
+### 1. Start Services
 
-3. **Accessing the Dashboard**:
-    - OpenObserve UI: http://localhost:5080
-    - Default credentials:
-        - Username: root@rustfs.com
-        - Password: rustfs123
+```bash
+cd .docker/openobserve-otel
+docker compose up -d
+```
 
-### Configuration
+### 2. Access Dashboard
 
-#### OpenObserve Configuration
+- **URL**: [http://localhost:5080](http://localhost:5080)
+- **Username**: `root@rustfs.com`
+- **Password**: `rustfs123`
 
-The OpenObserve service is configured with:
+## 🛠️ Configuration
 
-- Root user credentials
-- Data persistence through a volume mount
-- Memory cache enabled
-- Health checks
-- Exposed ports:
-    - 5080: HTTP API and UI
-    - 5081: OTLP gRPC
+### OpenObserve
 
-#### OpenTelemetry Collector Configuration
+- **Persistence**: Data is persisted to a Docker volume.
+- **Ports**:
+    - `5080`: HTTP API and UI
+    - `5081`: OTLP gRPC
 
-The collector is configured to:
+### OpenTelemetry Collector
 
-- Receive telemetry data via OTLP (HTTP and gRPC)
-- Collect logs from files
-- Process data in batches
-- Export data to OpenObserve
-- Manage memory usage
+- **Receivers**: OTLP (gRPC `4317`, HTTP `4318`)
+- **Exporters**: Sends data to OpenObserve.
 
-### Integration with Your Application
+## 🔗 Integration
 
-To send telemetry data from your application, configure your OpenTelemetry SDK to send data to:
+Configure your application to send OTLP data to the collector:
 
-- OTLP gRPC: `localhost:4317`
-- OTLP HTTP: `localhost:4318`
+- **Endpoint**: `http://localhost:4318` (HTTP) or `localhost:4317` (gRPC)
 
-For example, in a Rust application using the `rustfs-obs` library:
+Example for RustFS:
 
 ```bash
 export RUSTFS_OBS_ENDPOINT=http://localhost:4318
-export RUSTFS_OBS_SERVICE_NAME=yourservice
-export RUSTFS_OBS_SERVICE_VERSION=1.0.0
-export RUSTFS_OBS_ENVIRONMENT=development
+export RUSTFS_OBS_SERVICE_NAME=rustfs-node-1
 ```
-

.docker/openobserve-otel/README_ZH.md

@@ -5,71 +5,57 @@
 
 [English](README.md) | 中文
 
-## 中文
+本目录包含使用 OpenObserve 的**替代**可观测性技术栈配置。
 
-本目录包含搭建 OpenObserve 和 OpenTelemetry Collector 可观测性栈的配置文件。
+## ⚠️ 注意
 
-### 概述
+对于**推荐**的可观测性技术栈（Prometheus, Grafana, Tempo, Loki），请参阅 `../observability/`。
 
-此设置为应用程序提供了完整的可观测性解决方案：
+## 🌟 概览
 
-- **OpenObserve**：现代化、开源的可观测性平台，用于日志、指标和追踪。
-- **OpenTelemetry Collector**：收集和处理遥测数据，然后将其发送到 OpenObserve。
+OpenObserve 是一个轻量级、一体化的可观测性平台，在一个二进制文件中处理日志、指标和追踪。此设置非常适合：
+- 资源受限的环境。
+- 快速设置和测试。
+- 喜欢统一 UI 的用户。
 
-### 设置说明
+## 🚀 快速开始
 
-1. **前提条件**：
-    - 已安装 Docker 和 Docker Compose
-    - 足够的内存资源（建议至少 2GB）
-
-2. **启动服务**：
-   ```bash
-   cd .docker/openobserve-otel
-   docker compose -f docker-compose.yml  up -d
-   ```
-
-3. **访问仪表板**：
-    - OpenObserve UI：http://localhost:5080
-    - 默认凭据：
-        - 用户名：root@rustfs.com
-        - 密码：rustfs123
-
-### 配置
-
-#### OpenObserve 配置
-
-OpenObserve 服务配置：
-
-- 根用户凭据
-- 通过卷挂载实现数据持久化
-- 启用内存缓存
-- 健康检查
-- 暴露端口：
-    - 5080：HTTP API 和 UI
-    - 5081：OTLP gRPC
-
-#### OpenTelemetry Collector 配置
-
-收集器配置为：
-
-- 通过 OTLP（HTTP 和 gRPC）接收遥测数据
-- 从文件中收集日志
-- 批处理数据
-- 将数据导出到 OpenObserve
-- 管理内存使用
-
-### 与应用程序集成
-
-要从应用程序发送遥测数据，将 OpenTelemetry SDK 配置为发送数据到：
-
-- OTLP gRPC:`localhost:4317`
-- OTLP HTTP:`localhost:4318`
-
-例如，在使用 `rustfs-obs` 库的 Rust 应用程序中：
+### 1. 启动服务
 
 ```bash
-export RUSTFS_OBS_ENDPOINT=http://localhost:4317
-export RUSTFS_OBS_SERVICE_NAME=yourservice
-export RUSTFS_OBS_SERVICE_VERSION=1.0.0
-export RUSTFS_OBS_ENVIRONMENT=development
-```
+cd .docker/openobserve-otel
+docker compose up -d
+```
+
+### 2. 访问仪表盘
+
+- **URL**: [http://localhost:5080](http://localhost:5080)
+- **用户名**: `root@rustfs.com`
+- **密码**: `rustfs123`
+
+## 🛠️ 配置
+
+### OpenObserve
+
+- **持久化**: 数据持久化到 Docker 卷。
+- **端口**:
+    - `5080`: HTTP API 和 UI
+    - `5081`: OTLP gRPC
+
+### OpenTelemetry Collector
+
+- **接收器**: OTLP (gRPC `4317`, HTTP `4318`)
+- **导出器**: 将数据发送到 OpenObserve。
+
+## 🔗 集成
+
+配置您的应用程序将 OTLP 数据发送到收集器：
+
+- **端点**: `http://localhost:4318` (HTTP) 或 `localhost:4317` (gRPC)
+
+RustFS 示例：
+
+```bash
+export RUSTFS_OBS_ENDPOINT=http://localhost:4318
+export RUSTFS_OBS_SERVICE_NAME=rustfs-node-1
+```

.github/AGENTS.md

@@ -0,0 +1,30 @@
+# GitHub Workflow Instructions
+
+Applies to `.github/` and repository pull-request operations.
+
+## Pull Requests
+
+- PR titles and descriptions must be in English.
+- Use `.github/pull_request_template.md` for every PR body.
+- Keep all template section headings.
+- Use `N/A` for non-applicable sections.
+- Include verification commands in the PR details.
+- For `gh pr create` and `gh pr edit`, always write markdown body to a file and pass `--body-file`.
+- Do not use multiline inline `--body`; backticks and shell expansion can corrupt content or trigger unintended commands.
+- Recommended pattern:
+  - `cat > /tmp/pr_body.md <<'EOF'`
+  - `...markdown...`
+  - `EOF`
+  - `gh pr create ... --body-file /tmp/pr_body.md`
+
+## CI Alignment
+
+When changing CI-sensitive behavior, keep local validation aligned with `.github/workflows/ci.yml`.
+
+Current `test-and-lint` gate includes:
+
+- `cargo nextest run --all --exclude e2e_test`
+- `cargo test --all --doc`
+- `cargo fmt --all --check`
+- `cargo clippy --all-targets --all-features -- -D warnings`
+- `./scripts/check_layer_dependencies.sh`

.github/dependabot.yml

@@ -29,6 +29,8 @@ updates:
     ignore:
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
+      - dependency-name: "libunftp"
+        versions: [ "0.23.x" ]
     groups:
       s3s:
         update-types:

.github/workflows/build.yml

@@ -44,22 +44,6 @@ on:
       - "**/*.svg"
       - ".gitignore"
       - ".dockerignore"
-  pull_request:
-    branches: [ main ]
-    paths-ignore:
-      - "**.md"
-      - "**.txt"
-      - ".github/**"
-      - "docs/**"
-      - "deploy/**"
-      - "scripts/dev_*.sh"
-      - "LICENSE*"
-      - "README*"
-      - "**/*.png"
-      - "**/*.jpg"
-      - "**/*.svg"
-      - ".gitignore"
-      - ".dockerignore"
   schedule:
     - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
   workflow_dispatch:
@@ -161,42 +145,52 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     env:
-      RUSTFLAGS: ${{ matrix.cross == 'false' && '-C target-cpu=native' || '' }}
+      RUSTFLAGS: ${{ matrix.rustflags }}
     strategy:
       fail-fast: false
       matrix:
         include:
           # Linux builds
+          # Use x86-64-v2 (SSE4.2 baseline) instead of native to ensure distributed
+          # binaries run on older x86_64 CPUs (e.g. Intel Celeron/Atom, Synology NAS).
+          # See: https://github.com/rustfs/rustfs/issues/1838
           - os: ubicloud-standard-2
             target: x86_64-unknown-linux-musl
             cross: false
             platform: linux
+            rustflags: ''
           - os: ubicloud-standard-2
             target: aarch64-unknown-linux-musl
             cross: true
             platform: linux
+            rustflags: ''
           - os: ubicloud-standard-2
             target: x86_64-unknown-linux-gnu
             cross: false
             platform: linux
+            rustflags: ''
           - os: ubicloud-standard-2
             target: aarch64-unknown-linux-gnu
             cross: true
             platform: linux
+            rustflags: ''
           # macOS builds
           - os: macos-latest
             target: aarch64-apple-darwin
             cross: false
             platform: macos
+            rustflags: ''
           - os: macos-latest
             target: x86_64-apple-darwin
             cross: false
             platform: macos
+            rustflags: ''
           # Windows builds (temporarily disabled)
           - os: windows-latest
             target: x86_64-pc-windows-msvc
             cross: false
             platform: windows
+            rustflags: ''
           #- os: windows-latest
           #  target: aarch64-pc-windows-msvc
           #  cross: true

.github/workflows/ci.yml

@@ -19,7 +19,6 @@ on:
     branches: [ main ]
     paths-ignore:
       - "**.md"
-      - "**.txt"
       - "docs/**"
       - "deploy/**"
       - "scripts/dev_*.sh"
@@ -39,7 +38,6 @@ on:
     branches: [ main ]
     paths-ignore:
       - "**.md"
-      - "**.txt"
       - "docs/**"
       - "deploy/**"
       - "scripts/dev_*.sh"
@@ -93,6 +91,8 @@ jobs:
 
   typos:
     name: Typos
+    needs: skip-check
+    if: needs.skip-check.outputs.should_skip != 'true'
     runs-on: ubicloud-standard-2
     steps:
       - uses: actions/checkout@v6
@@ -118,9 +118,6 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - name: Install cargo-nextest
-        uses: taiki-e/install-action@nextest
-
       - name: Run tests
         run: |
           cargo nextest run --all --exclude e2e_test
@@ -132,9 +129,43 @@ jobs:
       - name: Run clippy lints
         run: cargo clippy --all-targets --all-features -- -D warnings
 
+      - name: Check layered dependencies
+        run: ./scripts/check_layer_dependencies.sh
+
+  build-rustfs-debug-binary:
+    name: Build RustFS Debug Binary
+    needs: skip-check
+    if: needs.skip-check.outputs.should_skip != 'true'
+    runs-on: ubicloud-standard-4
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-rustfs-debug-binary-${{ hashFiles('**/Cargo.lock') }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build debug binary
+        run: |
+          touch rustfs/build.rs
+          cargo build -p rustfs --bins --jobs 2
+
+      - name: Upload debug binary
+        uses: actions/upload-artifact@v6
+        with:
+          name: rustfs-debug-binary
+          path: target/debug/rustfs
+          if-no-files-found: error
+          retention-days: 1
+
   e2e-tests:
     name: End-to-End Tests
-    needs: skip-check
+    needs: [ skip-check, build-rustfs-debug-binary ]
     if: needs.skip-check.outputs.should_skip != 'true'
     runs-on: ubicloud-standard-2
     timeout-minutes: 30
@@ -147,13 +178,17 @@ jobs:
           rm -rf /tmp/rustfs
           rm -f /tmp/rustfs.log
 
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
+      - name: Download debug binary
+        uses: actions/download-artifact@v7
         with:
-          rust-version: stable
-          cache-shared-key: ci-e2e-${{ hashFiles('**/Cargo.lock') }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          name: rustfs-debug-binary
+          path: target/debug
+
+      - name: Make binary executable
+        run: chmod +x ./target/debug/rustfs
+
+      - name: Setup Rust toolchain for s3s-e2e installation
+        uses: dtolnay/rust-toolchain@stable
 
       - name: Install s3s-e2e test tool
         uses: taiki-e/cache-cargo-install-action@v2
@@ -162,12 +197,6 @@ jobs:
           git: https://github.com/s3s-project/s3s.git
           rev: 4a04a670cf41274d9be9ab65dc36f4aa3f92fbad
 
-      - name: Build debug binary
-        run: |
-          touch rustfs/build.rs
-          # Limit concurrency to prevent OOM
-          cargo build -p rustfs --bins --jobs 2
-
       - name: Run end-to-end tests
         run: |
           s3s-e2e --version
@@ -180,3 +209,39 @@ jobs:
           name: e2e-test-logs-${{ github.run_number }}
           path: /tmp/rustfs.log
           retention-days: 3
+
+  s3-implemented-tests:
+    name: S3 Implemented Tests
+    needs: [ skip-check, build-rustfs-debug-binary ]
+    if: needs.skip-check.outputs.should_skip != 'true'
+    runs-on: ubicloud-standard-4
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Download debug binary
+        uses: actions/download-artifact@v7
+        with:
+          name: rustfs-debug-binary
+          path: target/debug
+
+      - name: Make binary executable
+        run: chmod +x ./target/debug/rustfs
+
+      - name: Run implemented s3-tests
+        run: |
+          DEPLOY_MODE=binary \
+          RUSTFS_BINARY=./target/debug/rustfs \
+          TEST_MODE=single \
+          MAXFAIL=1 \
+          ./scripts/s3-tests/run.sh
+
+      - name: Upload s3 test artifacts
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: s3tests-implemented-${{ github.run_number }}
+          path: artifacts/s3tests-single/**
+          if-no-files-found: ignore
+          retention-days: 3

.github/workflows/docker.yml

@@ -66,6 +66,7 @@ env:
   CARGO_TERM_COLOR: always
   REGISTRY_DOCKERHUB: rustfs/rustfs
   REGISTRY_GHCR: ghcr.io/${{ github.repository }}
+  REGISTRY_QUAY: quay.io/${{ secrets.QUAY_USERNAME }}/rustfs
   DOCKER_PLATFORMS: linux/amd64,linux/arm64
 
 jobs:
@@ -162,14 +163,7 @@ jobs:
                 if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
                   build_type="prerelease"
                   is_prerelease=true
-                  # TODO: Temporary change - currently allows alpha versions to also create latest tags
-                  # After the version is stable, you need to remove the following line and restore the original logic (latest is created only for stable versions)
-                  if [[ "$version" == *"alpha"* ]]; then
-                    create_latest=true
-                     echo "🧪 Building Docker image for prerelease: $version (temporarily allowing creation of latest tag)"
-                  else
-                    echo "🧪 Building Docker image for prerelease: $version"
-                  fi
+                  echo "🧪 Building Docker image for prerelease: $version"
                 else
                   build_type="release"
                   create_latest=true
@@ -215,14 +209,7 @@ jobs:
               v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
                 build_type="prerelease"
                 is_prerelease=true
-                # TODO: Temporary change - currently allows alpha versions to also create latest tags
-                # After the version is stable, you need to remove the if block below and restore the original logic.
-                if [[ "$input_version" == *"alpha"* ]]; then
-                  create_latest=true
-                   echo "🧪 Building with prerelease version: $input_version (temporarily allowing creation of latest tag)"
-                else
-                  echo "🧪 Building with prerelease version: $input_version"
-                fi
+                echo "🧪 Building with prerelease version: $input_version"
                 ;;
               # Release versions (match after prereleases, more general)
               v[0-9]*|[0-9]*.*.*)
@@ -287,12 +274,19 @@ jobs:
           username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      # - name: Login to GitHub Container Registry
-      #   uses: docker/login-action@v3
-      #   with:
-      #     registry: ghcr.io
-      #     username: ${{ github.actor }}
-      #     password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GHCR_USERNAME }}
+          password: ${{ secrets.GHCR_PASSWORD }}
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -338,14 +332,15 @@ jobs:
 
           # Generate tags based on build type
           # Only support release and prerelease builds (no development builds)
-          TAGS="${{ env.REGISTRY_DOCKERHUB }}:${VERSION}${VARIANT_SUFFIX}"
+          TAG_BASE="${VERSION}${VARIANT_SUFFIX}"
+          TAGS="${{ env.REGISTRY_DOCKERHUB }}:$TAG_BASE,${{ env.REGISTRY_GHCR }}:$TAG_BASE,${{ env.REGISTRY_QUAY }}:$TAG_BASE"
 
           # Add channel tags for prereleases and latest for stable
           if [[ "$CREATE_LATEST" == "true" ]]; then
             # TODO: Temporary change - the current alpha version will also create the latest tag
             # After the version is stabilized, the logic here remains unchanged, but the upstream CREATE_LATEST setting needs to be restored.
             # Stable release (and temporary alpha versions)
-            TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX}"
+            TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_GHCR }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_QUAY }}:latest${VARIANT_SUFFIX}"
           elif [[ "$BUILD_TYPE" == "prerelease" ]]; then
             # Prerelease channel tags (alpha, beta, rc)
             if [[ "$VERSION" == *"alpha"* ]]; then
@@ -357,7 +352,7 @@ jobs:
             fi
 
             if [[ -n "$CHANNEL" ]]; then
-              TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}${VARIANT_SUFFIX}"
+              TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}${VARIANT_SUFFIX},${{ env.REGISTRY_GHCR }}:${CHANNEL}${VARIANT_SUFFIX},${{ env.REGISTRY_QUAY }}:${CHANNEL}${VARIANT_SUFFIX}"
             fi
           fi
 
@@ -441,10 +436,8 @@ jobs:
             "prerelease")
               echo "🧪 Prerelease Docker image has been built with ${VERSION} tags"
               echo "⚠️  This is a prerelease image - use with caution"
-              # TODO: Temporary change - alpha versions currently create the latest tag
-              # After the version is stable, you need to restore the following prompt information
-              if [[ "$VERSION" == *"alpha"* ]] && [[ "$CREATE_LATEST" == "true" ]]; then
-                echo "🏷️  Latest tag has been created for alpha version (temporary measures)"
+              if [[ "$CREATE_LATEST" == "true" ]]; then
+                echo "🏷️  Latest tag has been explicitly created for prerelease"
               else
                 echo "🚫 Latest tag NOT created for prerelease"
               fi

.github/workflows/e2e-s3tests.yml

@@ -50,6 +50,11 @@ env:
 
   RUST_LOG: info
   PLATFORM: linux/amd64
+  BUILDX_CACHE_SCOPE: rustfs-e2e-s3tests-source
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs['test-mode'] || 'single' }}
+  cancel-in-progress: true
 
 defaults:
   run:
@@ -57,12 +62,25 @@ defaults:
 
 jobs:
   s3tests-single:
-    if: github.event.inputs.test-mode == 'single'
+    if: github.event.inputs['test-mode'] == 'single'
     runs-on: ubicloud-standard-2
     timeout-minutes: 120
     steps:
       - uses: actions/checkout@v6
 
+      - name: Cache pip downloads
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-e2e-s3tests-${{ hashFiles('.github/workflows/e2e-s3tests.yml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-e2e-s3tests-
+
+      - name: Install Python tools
+        run: |
+          python3 -m pip install --user --upgrade pip awscurl tox
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
       - name: Enable buildx
         uses: docker/setup-buildx-action@v3
 
@@ -70,8 +88,8 @@ jobs:
         run: |
           DOCKER_BUILDKIT=1 docker buildx build --load \
             --platform ${PLATFORM} \
-            --cache-from type=gha \
-            --cache-to type=gha,mode=max \
+            --cache-from type=gha,scope=${BUILDX_CACHE_SCOPE} \
+            --cache-to type=gha,mode=max,scope=${BUILDX_CACHE_SCOPE} \
             -t rustfs-ci \
             -f Dockerfile.source .
 
@@ -121,9 +139,6 @@ jobs:
 
       - name: Provision s3-tests alt user (required by suite)
         run: |
-          python3 -m pip install --user --upgrade pip awscurl
-          export PATH="$HOME/.local/bin:$PATH"
-
           # Admin API requires AWS SigV4 signing. awscurl is used by RustFS codebase as well.
           awscurl \
             --service s3 \
@@ -156,8 +171,6 @@ jobs:
 
       - name: Prepare s3-tests
         run: |
-          python3 -m pip install --user --upgrade pip tox
-          export PATH="$HOME/.local/bin:$PATH"
           git clone --depth 1 https://github.com/ceph/s3-tests.git s3-tests
 
       - name: Run ceph s3-tests (debug friendly)
@@ -211,12 +224,25 @@ jobs:
           path: artifacts/**
 
   s3tests-multi:
-    if: github.event_name == 'workflow_dispatch' && github.event.inputs.test-mode == 'multi'
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs['test-mode'] == 'multi'
     runs-on: ubicloud-standard-2
     timeout-minutes: 150
     steps:
       - uses: actions/checkout@v6
 
+      - name: Cache pip downloads
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-e2e-s3tests-${{ hashFiles('.github/workflows/e2e-s3tests.yml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-e2e-s3tests-
+
+      - name: Install Python tools
+        run: |
+          python3 -m pip install --user --upgrade pip awscurl tox
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
       - name: Enable buildx
         uses: docker/setup-buildx-action@v3
 
@@ -224,8 +250,8 @@ jobs:
         run: |
           DOCKER_BUILDKIT=1 docker buildx build --load \
             --platform ${PLATFORM} \
-            --cache-from type=gha \
-            --cache-to type=gha,mode=max \
+            --cache-from type=gha,scope=${BUILDX_CACHE_SCOPE} \
+            --cache-to type=gha,mode=max,scope=${BUILDX_CACHE_SCOPE} \
             -t rustfs-ci \
             -f Dockerfile.source .
 
@@ -337,9 +363,6 @@ jobs:
 
       - name: Provision s3-tests alt user (required by suite)
         run: |
-          python3 -m pip install --user --upgrade pip awscurl
-          export PATH="$HOME/.local/bin:$PATH"
-
           awscurl \
             --service s3 \
             --region "${S3_REGION}" \
@@ -368,8 +391,6 @@ jobs:
 
       - name: Prepare s3-tests
         run: |
-          python3 -m pip install --user --upgrade pip tox
-          export PATH="$HOME/.local/bin:$PATH"
           git clone --depth 1 https://github.com/ceph/s3-tests.git s3-tests
 
       - name: Run ceph s3-tests (multi, debug friendly)

.github/workflows/nix-flake-update.yml

@@ -0,0 +1,61 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Update Nix Flake
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 0' # Weekly on Sundays
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  update-flake:
+    name: Update flake.lock
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Install Nix
+        uses: DeterminateSystems/determinate-nix-action@v3
+
+      - name: Check Nix flake inputs
+        uses: DeterminateSystems/flake-checker-action@v12
+
+      - name: Update flake.lock
+        id: update
+        uses: DeterminateSystems/update-flake-lock@main
+        with:
+          pr-title: "chore(deps): update flake.lock"
+          pr-labels: |
+            dependencies
+            nix
+            automated
+          commit-msg: "chore(deps): update flake.lock"
+          pr-reviewers: houseme, heihutu
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Log PR details
+        if: steps.update.outputs.pull-request-number
+        run: |
+          echo "Pull Request created: ${{ steps.update.outputs.pull-request-number }}"

.github/workflows/nix.yml

@@ -0,0 +1,84 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Nix CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'flake.nix'
+      - 'flake.lock'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
+      - '.github/workflows/nix.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'flake.nix'
+      - 'flake.lock'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
+      - '.github/workflows/nix.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+permissions:
+  contents: read
+
+jobs:
+  nix-validation:
+    name: Nix Build & Check
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: Setup Magic Nix Cache
+        uses: DeterminateSystems/magic-nix-cache-action@v13
+
+      - name: Setup Flake Checker
+        uses: DeterminateSystems/flake-checker-action@v12
+
+      - name: Verify Flake
+        run: |
+          echo "Checking flake structure and evaluation..."
+          nix flake show
+          nix flake check --print-build-logs
+
+      - name: Build RustFS
+        run: |
+          echo "Building the default package..."
+          nix build .#default --print-build-logs
+
+      - name: Test Binary
+        run: |
+          echo "Verifying the built binary..."
+          if [ -x "./result/bin/rustfs" ]; then
+              ./result/bin/rustfs --help
+              echo "Binary verification successful."
+          else
+              echo "Error: Binary not found or not executable at ./result/bin/rustfs"
+              exit 1
+          fi

.gitignore

@@ -2,6 +2,7 @@
 .DS_Store
 .idea
 .vscode
+.cursor
 .direnv/
 /test
 /logs
@@ -39,4 +40,9 @@ artifacts/
 PR_DESCRIPTION.md
 IMPLEMENTATION_PLAN.md
 scripts/s3-tests/selected_tests.txt
-docs
+docs
+
+# nix stuff
+result*
+*.gz
+rustfs-webdav.code-workspace

.rules.md

@@ -1,702 +0,0 @@
-# RustFS Project AI Coding Rules
-
-## 🚨🚨🚨 CRITICAL DEVELOPMENT RULES - ZERO TOLERANCE 🚨🚨🚨
-
-### ⛔️ ABSOLUTE PROHIBITION: NEVER COMMIT DIRECTLY TO MASTER/MAIN BRANCH ⛔️
-
-**🔥 THIS IS THE MOST CRITICAL RULE - VIOLATION WILL RESULT IN IMMEDIATE REVERSAL 🔥**
-
-- **🚫 ZERO DIRECT COMMITS TO MAIN/MASTER BRANCH - ABSOLUTELY FORBIDDEN**
-- **🚫 ANY DIRECT COMMIT TO MAIN BRANCH MUST BE IMMEDIATELY REVERTED**
-- **🚫 NO EXCEPTIONS FOR HOTFIXES, EMERGENCIES, OR URGENT CHANGES**
-- **🚫 NO EXCEPTIONS FOR SMALL CHANGES, TYPOS, OR DOCUMENTATION UPDATES**
-- **🚫 NO EXCEPTIONS FOR ANYONE - MAINTAINERS, CONTRIBUTORS, OR ADMINS**
-
-### 📋 MANDATORY WORKFLOW - STRICTLY ENFORCED
-
-**EVERY SINGLE CHANGE MUST FOLLOW THIS WORKFLOW:**
-
-1. **Check current branch**: `git branch` (MUST NOT be on main/master)
-2. **Switch to main**: `git checkout main`
-3. **Pull latest**: `git pull origin main`
-4. **Create feature branch**: `git checkout -b feat/your-feature-name`
-5. **Make changes ONLY on feature branch**
-6. **Test thoroughly before committing**
-7. **Commit and push to feature branch**: `git push origin feat/your-feature-name`
-8. **Create Pull Request**: Use `gh pr create` (MANDATORY)
-9. **Wait for PR approval**: NO self-merging allowed
-10. **Merge through GitHub interface**: ONLY after approval
-
-### 🔒 ENFORCEMENT MECHANISMS
-
-- **Branch protection rules**: Main branch is protected
-- **Pre-commit hooks**: Will block direct commits to main
-- **CI/CD checks**: All PRs must pass before merging
-- **Code review requirement**: At least one approval needed
-- **Automated reversal**: Direct commits to main will be automatically reverted
-
-## 🎯 Core AI Development Principles
-
-### Five Execution Steps
-
-#### 1. Task Analysis and Planning
-- **Clear Objectives**: Deeply understand task requirements and expected results before starting coding
-- **Plan Development**: List specific files, components, and functions that need modification, explaining the reasons for changes
-- **Risk Assessment**: Evaluate the impact of changes on existing functionality, develop rollback plans
-
-#### 2. Precise Code Location
-- **File Identification**: Determine specific files and line numbers that need modification
-- **Impact Analysis**: Avoid modifying irrelevant files, clearly state the reason for each file modification
-- **Minimization Principle**: Unless explicitly required by the task, do not create new abstraction layers or refactor existing code
-
-#### 3. Minimal Code Changes
-- **Focus on Core**: Only write code directly required by the task
-- **Avoid Redundancy**: Do not add unnecessary logs, comments, tests, or error handling
-- **Isolation**: Ensure new code does not interfere with existing functionality, maintain code independence
-
-#### 4. Strict Code Review
-- **Correctness Check**: Verify the correctness and completeness of code logic
-- **Style Consistency**: Ensure code conforms to established project coding style
-- **Side Effect Assessment**: Evaluate the impact of changes on downstream systems
-
-#### 5. Clear Delivery Documentation
-- **Change Summary**: Detailed explanation of all modifications and reasons
-- **File List**: List all modified files and their specific changes
-- **Risk Statement**: Mark any assumptions or potential risk points
-
-### Core Principles
-- **🎯 Precise Execution**: Strictly follow task requirements, no arbitrary innovation
-- **⚡ Efficient Development**: Avoid over-design, only do necessary work
-- **🛡️ Safe and Reliable**: Always follow development processes, ensure code quality and system stability
-- **🔒 Cautious Modification**: Only modify when clearly knowing what needs to be changed and having confidence
-
-### Additional AI Behavior Rules
-
-1. **Use English for all code comments and documentation** - All comments, variable names, function names, documentation, and user-facing text in code should be in English
-2. **Clean up temporary scripts after use** - Any temporary scripts, test files, or helper files created during AI work should be removed after task completion
-3. **Only make confident modifications** - Do not make speculative changes or "convenient" modifications outside the task scope. If uncertain about a change, ask for clarification rather than guessing
-
-## Project Overview
-
-RustFS is a high-performance distributed object storage system written in Rust, compatible with S3 API. The project adopts a modular architecture, supporting erasure coding storage, multi-tenant management, observability, and other enterprise-level features.
-
-## Core Architecture Principles
-
-### 1. Modular Design
-
-- Project uses Cargo workspace structure, containing multiple independent crates
-- Core modules: `rustfs` (main service), `ecstore` (erasure coding storage), `common` (shared components)
-- Functional modules: `iam` (identity management), `madmin` (management interface), `crypto` (encryption), etc.
-- Tool modules: `cli` (command line tool), `crates/*` (utility libraries)
-
-### 2. Asynchronous Programming Pattern
-
-- Comprehensive use of `tokio` async runtime
-- Prioritize `async/await` syntax
-- Use `async-trait` for async methods in traits
-- Avoid blocking operations, use `spawn_blocking` when necessary
-
-### 3. Error Handling Strategy
-
-- **Use modular, type-safe error handling with `thiserror`**
-- Each module should define its own error type using `thiserror::Error` derive macro
-- Support error chains and context information through `#[from]` and `#[source]` attributes
-- Use `Result<T>` type aliases for consistency within each module
-- Error conversion between modules should use explicit `From` implementations
-- Follow the pattern: `pub type Result<T> = core::result::Result<T, Error>`
-- Use `#[error("description")]` attributes for clear error messages
-- Support error downcasting when needed through `other()` helper methods
-- Implement `Clone` for errors when required by the domain logic
-
-## Code Style Guidelines
-
-### 1. Formatting Configuration
-
-```toml
-max_width = 130
-fn_call_width = 90
-single_line_let_else_max_width = 100
-```
-
-### 2. **🔧 MANDATORY Code Formatting Rules**
-
-**CRITICAL**: All code must be properly formatted before committing. This project enforces strict formatting standards to maintain code consistency and readability.
-
-#### Pre-commit Requirements (MANDATORY)
-
-Before every commit, you **MUST**:
-
-1. **Format your code**:
-   ```bash
-   cargo fmt --all
-   ```
-
-2. **Verify formatting**:
-   ```bash
-   cargo fmt --all --check
-   ```
-
-3. **Pass clippy checks**:
-   ```bash
-   cargo clippy --all-targets --all-features -- -D warnings
-   ```
-
-4. **Ensure compilation**:
-   ```bash
-   cargo check --all-targets
-   ```
-
-#### Quick Commands
-
-Use these convenient Makefile targets for common tasks:
-
-```bash
-# Format all code
-make fmt
-
-# Check if code is properly formatted
-make fmt-check
-
-# Run clippy checks
-make clippy
-
-# Run compilation check
-make check
-
-# Run tests
-make test
-
-# Run all pre-commit checks (format + clippy + check + test)
-make pre-commit
-
-# Setup git hooks (one-time setup)
-make setup-hooks
-```
-
-### 3. Naming Conventions
-
-- Use `snake_case` for functions, variables, modules
-- Use `PascalCase` for types, traits, enums
-- Constants use `SCREAMING_SNAKE_CASE`
-- Global variables prefix `GLOBAL_`, e.g., `GLOBAL_Endpoints`
-- Use meaningful and descriptive names for variables, functions, and methods
-- Avoid meaningless names like `temp`, `data`, `foo`, `bar`, `test123`
-- Choose names that clearly express the purpose and intent
-
-### 4. Type Declaration Guidelines
-
-- **Prefer type inference over explicit type declarations** when the type is obvious from context
-- Let the Rust compiler infer types whenever possible to reduce verbosity and improve maintainability
-- Only specify types explicitly when:
-  - The type cannot be inferred by the compiler
-  - Explicit typing improves code clarity and readability
-  - Required for API boundaries (function signatures, public struct fields)
-  - Needed to resolve ambiguity between multiple possible types
-
-### 5. Documentation Comments
-
-- Public APIs must have documentation comments
-- Use `///` for documentation comments
-- Complex functions add `# Examples` and `# Parameters` descriptions
-- Error cases use `# Errors` descriptions
-- Always use English for all comments and documentation
-- Avoid meaningless comments like "debug 111" or placeholder text
-
-### 6. Import Guidelines
-
-- Standard library imports first
-- Third-party crate imports in the middle
-- Project internal imports last
-- Group `use` statements with blank lines between groups
-
-## Asynchronous Programming Guidelines
-
-### 1. Trait Definition
-
-```rust
-#[async_trait::async_trait]
-pub trait StorageAPI: Send + Sync {
-    async fn get_object(&self, bucket: &str, object: &str) -> Result<ObjectInfo>;
-}
-```
-
-### 2. Error Handling
-
-```rust
-// Use ? operator to propagate errors
-async fn example_function() -> Result<()> {
-    let data = read_file("path").await?;
-    process_data(data).await?;
-    Ok(())
-}
-```
-
-### 3. Concurrency Control
-
-- Use `Arc` and `Mutex`/`RwLock` for shared state management
-- Prioritize async locks from `tokio::sync`
-- Avoid holding locks for long periods
-
-## Logging and Tracing Guidelines
-
-### 1. Tracing Usage
-
-```rust
-#[tracing::instrument(skip(self, data))]
-async fn process_data(&self, data: &[u8]) -> Result<()> {
-    info!("Processing {} bytes", data.len());
-    // Implementation logic
-}
-```
-
-### 2. Log Levels
-
-- `error!`: System errors requiring immediate attention
-- `warn!`: Warning information that may affect functionality
-- `info!`: Important business information
-- `debug!`: Debug information for development use
-- `trace!`: Detailed execution paths
-
-### 3. Structured Logging
-
-```rust
-info!(
-    counter.rustfs_api_requests_total = 1_u64,
-    key_request_method = %request.method(),
-    key_request_uri_path = %request.uri().path(),
-    "API request processed"
-);
-```
-
-## Error Handling Guidelines
-
-### 1. Error Type Definition
-
-```rust
-// Use thiserror for module-specific error types
-#[derive(thiserror::Error, Debug)]
-pub enum MyError {
-    #[error("IO error: {0}")]
-    Io(#[from] std::io::Error),
-
-    #[error("Storage error: {0}")]
-    Storage(#[from] ecstore::error::StorageError),
-
-    #[error("Custom error: {message}")]
-    Custom { message: String },
-
-    #[error("File not found: {path}")]
-    FileNotFound { path: String },
-
-    #[error("Invalid configuration: {0}")]
-    InvalidConfig(String),
-}
-
-// Provide Result type alias for the module
-pub type Result<T> = core::result::Result<T, MyError>;
-```
-
-### 2. Error Helper Methods
-
-```rust
-impl MyError {
-    /// Create error from any compatible error type
-    pub fn other<E>(error: E) -> Self
-    where
-        E: Into<Box<dyn std::error::Error + Send + Sync>>,
-    {
-        MyError::Io(std::io::Error::other(error))
-    }
-}
-```
-
-### 3. Error Context and Propagation
-
-```rust
-// Use ? operator for clean error propagation
-async fn example_function() -> Result<()> {
-    let data = read_file("path").await?;
-    process_data(data).await?;
-    Ok(())
-}
-
-// Add context to errors
-fn process_with_context(path: &str) -> Result<()> {
-    std::fs::read(path)
-        .map_err(|e| MyError::Custom {
-            message: format!("Failed to read {}: {}", path, e)
-        })?;
-    Ok(())
-}
-```
-
-## Performance Optimization Guidelines
-
-### 1. Memory Management
-
-- Use `Bytes` instead of `Vec<u8>` for zero-copy operations
-- Avoid unnecessary cloning, use reference passing
-- Use `Arc` for sharing large objects
-
-### 2. Concurrency Optimization
-
-```rust
-// Use join_all for concurrent operations
-let futures = disks.iter().map(|disk| disk.operation());
-let results = join_all(futures).await;
-```
-
-### 3. Caching Strategy
-
-- Use `LazyLock` for global caching
-- Implement LRU cache to avoid memory leaks
-
-## Testing Guidelines
-
-### 1. Unit Tests
-
-```rust
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use test_case::test_case;
-
-    #[tokio::test]
-    async fn test_async_function() {
-        let result = async_function().await;
-        assert!(result.is_ok());
-    }
-
-    #[test_case("input1", "expected1")]
-    #[test_case("input2", "expected2")]
-    fn test_with_cases(input: &str, expected: &str) {
-        assert_eq!(function(input), expected);
-    }
-}
-```
-
-### 2. Integration Tests
-
-- Use `e2e_test` module for end-to-end testing
-- Simulate real storage environments
-
-### 3. Test Quality Standards
-
-- Write meaningful test cases that verify actual functionality
-- Avoid placeholder or debug content like "debug 111", "test test", etc.
-- Use descriptive test names that clearly indicate what is being tested
-- Each test should have a clear purpose and verify specific behavior
-- Test data should be realistic and representative of actual use cases
-
-## Cross-Platform Compatibility Guidelines
-
-### 1. CPU Architecture Compatibility
-
-- **Always consider multi-platform and different CPU architecture compatibility** when writing code
-- Support major architectures: x86_64, aarch64 (ARM64), and other target platforms
-- Use conditional compilation for architecture-specific code:
-
-```rust
-#[cfg(target_arch = "x86_64")]
-fn optimized_x86_64_function() { /* x86_64 specific implementation */ }
-
-#[cfg(target_arch = "aarch64")]
-fn optimized_aarch64_function() { /* ARM64 specific implementation */ }
-
-#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
-fn generic_function() { /* Generic fallback implementation */ }
-```
-
-### 2. Platform-Specific Dependencies
-
-- Use feature flags for platform-specific dependencies
-- Provide fallback implementations for unsupported platforms
-- Test on multiple architectures in CI/CD pipeline
-
-### 3. Endianness Considerations
-
-- Use explicit byte order conversion when dealing with binary data
-- Prefer `to_le_bytes()`, `from_le_bytes()` for consistent little-endian format
-- Use `byteorder` crate for complex binary format handling
-
-### 4. SIMD and Performance Optimizations
-
-- Use portable SIMD libraries like `wide` or `packed_simd`
-- Provide fallback implementations for non-SIMD architectures
-- Use runtime feature detection when appropriate
-
-## Security Guidelines
-
-### 1. Memory Safety
-
-- Disable `unsafe` code (workspace.lints.rust.unsafe_code = "deny")
-- Use `rustls` instead of `openssl`
-
-### 2. Authentication and Authorization
-
-```rust
-// Use IAM system for permission checks
-let identity = iam.authenticate(&access_key, &secret_key).await?;
-iam.authorize(&identity, &action, &resource).await?;
-```
-
-## Configuration Management Guidelines
-
-### 1. Environment Variables
-
-- Use `RUSTFS_` prefix
-- Support both configuration files and environment variables
-- Provide reasonable default values
-
-### 2. Configuration Structure
-
-```rust
-#[derive(Debug, Deserialize, Clone)]
-pub struct Config {
-    pub address: String,
-    pub volumes: String,
-    #[serde(default)]
-    pub console_enable: bool,
-}
-```
-
-## Dependency Management Guidelines
-
-### 1. Workspace Dependencies
-
-- Manage versions uniformly at workspace level
-- Use `workspace = true` to inherit configuration
-
-### 2. Feature Flags
-
-```rust
-[features]
-default = ["file"]
-gpu = ["dep:nvml-wrapper"]
-kafka = ["dep:rdkafka"]
-```
-
-## Deployment and Operations Guidelines
-
-### 1. Containerization
-
-- Provide Dockerfile and docker-compose configuration
-- Support multi-stage builds to optimize image size
-
-### 2. Observability
-
-- Integrate OpenTelemetry for distributed tracing
-- Support Prometheus metrics collection
-- Provide Grafana dashboards
-
-### 3. Health Checks
-
-```rust
-// Implement health check endpoint
-async fn health_check() -> Result<HealthStatus> {
-    // Check component status
-}
-```
-
-## Code Review Checklist
-
-### 1. **Code Formatting and Quality (MANDATORY)**
-
-- [ ] **Code is properly formatted** (`cargo fmt --all --check` passes)
-- [ ] **All clippy warnings are resolved** (`cargo clippy --all-targets --all-features -- -D warnings` passes)
-- [ ] **Code compiles successfully** (`cargo check --all-targets` passes)
-- [ ] **Pre-commit hooks are working** and all checks pass
-- [ ] **No formatting-related changes** mixed with functional changes (separate commits)
-
-### 2. Functionality
-
-- [ ] Are all error cases properly handled?
-- [ ] Is there appropriate logging?
-- [ ] Is there necessary test coverage?
-
-### 3. Performance
-
-- [ ] Are unnecessary memory allocations avoided?
-- [ ] Are async operations used correctly?
-- [ ] Are there potential deadlock risks?
-
-### 4. Security
-
-- [ ] Are input parameters properly validated?
-- [ ] Are there appropriate permission checks?
-- [ ] Is information leakage avoided?
-
-### 5. Cross-Platform Compatibility
-
-- [ ] Does the code work on different CPU architectures (x86_64, aarch64)?
-- [ ] Are platform-specific features properly gated with conditional compilation?
-- [ ] Is byte order handling correct for binary data?
-- [ ] Are there appropriate fallback implementations for unsupported platforms?
-
-### 6. Code Commits and Documentation
-
-- [ ] Does it comply with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)?
-- [ ] Are commit messages concise and under 72 characters for the title line?
-- [ ] Commit titles should be concise and in English, avoid Chinese
-- [ ] Is PR description provided in copyable markdown format for easy copying?
-
-## Common Patterns and Best Practices
-
-### 1. Resource Management
-
-```rust
-// Use RAII pattern for resource management
-pub struct ResourceGuard {
-    resource: Resource,
-}
-
-impl Drop for ResourceGuard {
-    fn drop(&mut self) {
-        // Clean up resources
-    }
-}
-```
-
-### 2. Dependency Injection
-
-```rust
-// Use dependency injection pattern
-pub struct Service {
-    config: Arc<Config>,
-    storage: Arc<dyn StorageAPI>,
-}
-```
-
-### 3. Graceful Shutdown
-
-```rust
-// Implement graceful shutdown
-async fn shutdown_gracefully(shutdown_rx: &mut Receiver<()>) {
-    tokio::select! {
-        _ = shutdown_rx.recv() => {
-            info!("Received shutdown signal");
-            // Perform cleanup operations
-        }
-        _ = tokio::time::sleep(SHUTDOWN_TIMEOUT) => {
-            warn!("Shutdown timeout reached");
-        }
-    }
-}
-```
-
-## Domain-Specific Guidelines
-
-### 1. Storage Operations
-
-- All storage operations must support erasure coding
-- Implement read/write quorum mechanisms
-- Support data integrity verification
-
-### 2. Network Communication
-
-- Use gRPC for internal service communication
-- HTTP/HTTPS support for S3-compatible API
-- Implement connection pooling and retry mechanisms
-
-### 3. Metadata Management
-
-- Use FlatBuffers for serialization
-- Support version control and migration
-- Implement metadata caching
-
-## Branch Management and Development Workflow
-
-### Branch Management
-
-- **🚨 CRITICAL: NEVER modify code directly on main or master branch - THIS IS ABSOLUTELY FORBIDDEN 🚨**
-- **⚠️ ANY DIRECT COMMITS TO MASTER/MAIN WILL BE REJECTED AND MUST BE REVERTED IMMEDIATELY ⚠️**
-- **🔒 ALL CHANGES MUST GO THROUGH PULL REQUESTS - NO DIRECT COMMITS TO MAIN UNDER ANY CIRCUMSTANCES 🔒**
-- **Always work on feature branches - NO EXCEPTIONS**
-- Always check the .rules.md file before starting to ensure you understand the project guidelines
-- **MANDATORY workflow for ALL changes:**
-   1. `git checkout main` (switch to main branch)
-   2. `git pull` (get latest changes)
-   3. `git checkout -b feat/your-feature-name` (create and switch to feature branch)
-   4. Make your changes ONLY on the feature branch
-   5. Test thoroughly before committing
-   6. Commit and push to the feature branch
-   7. **Create a pull request for code review - THIS IS THE ONLY WAY TO MERGE TO MAIN**
-   8. **Wait for PR approval before merging - NEVER merge your own PRs without review**
-- Use descriptive branch names following the pattern: `feat/feature-name`, `fix/issue-name`, `refactor/component-name`, etc.
-- **Double-check current branch before ANY commit: `git branch` to ensure you're NOT on main/master**
-- **Pull Request Requirements:**
-  - All changes must be submitted via PR regardless of size or urgency
-  - PRs must include comprehensive description and testing information
-  - PRs must pass all CI/CD checks before merging
-  - PRs require at least one approval from code reviewers
-  - Even hotfixes and emergency changes must go through PR process
-- **Enforcement:**
-  - Main branch should be protected with branch protection rules
-  - Direct pushes to main should be blocked by repository settings
-  - Any accidental direct commits to main must be immediately reverted via PR
-
-### Development Workflow
-
-## 🎯 **Core Development Principles**
-
-- **🔴 Every change must be precise - don't modify unless you're confident**
-  - Carefully analyze code logic and ensure complete understanding before making changes
-  - When uncertain, prefer asking users or consulting documentation over blind modifications
-  - Use small iterative steps, modify only necessary parts at a time
-  - Evaluate impact scope before changes to ensure no new issues are introduced
-
-- **🚀 GitHub PR creation prioritizes gh command usage**
-  - Prefer using `gh pr create` command to create Pull Requests
-  - Avoid having users manually create PRs through web interface
-  - Provide clear and professional PR titles and descriptions
-  - Using `gh` commands ensures better integration and automation
-
-## 📝 **Code Quality Requirements**
-
-- Use English for all code comments, documentation, and variable names
-- Write meaningful and descriptive names for variables, functions, and methods
-- Avoid meaningless test content like "debug 111" or placeholder values
-- Before each change, carefully read the existing code to ensure you understand the code structure and implementation, do not break existing logic implementation, do not introduce new issues
-- Ensure each change provides sufficient test cases to guarantee code correctness
-- Do not arbitrarily modify numbers and constants in test cases, carefully analyze their meaning to ensure test case correctness
-- When writing or modifying tests, check existing test cases to ensure they have scientific naming and rigorous logic testing, if not compliant, modify test cases to ensure scientific and rigorous testing
-- **Before committing any changes, run `cargo clippy --all-targets --all-features -- -D warnings` to ensure all code passes Clippy checks**
-- After each development completion, first git add . then git commit -m "feat: feature description" or "fix: issue description", ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
-- **Keep commit messages concise and under 72 characters** for the title line, use body for detailed explanations if needed
-- After each development completion, first git push to remote repository
-- After each change completion, summarize the changes, do not create summary files, provide a brief change description, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
-- Provide change descriptions needed for PR in the conversation, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
-- **Always provide PR descriptions in English** after completing any changes, including:
-  - Clear and concise title following Conventional Commits format
-  - Detailed description of what was changed and why
-  - List of key changes and improvements
-  - Any breaking changes or migration notes if applicable
-  - Testing information and verification steps
-- **Provide PR descriptions in copyable markdown format** enclosed in code blocks for easy one-click copying
-
-## 🚫 AI Documentation Generation Restrictions
-
-### Forbidden Summary Documents
-
-- **Strictly forbidden to create any form of AI-generated summary documents**
-- **Do not create documents containing large amounts of emoji, detailed formatting tables and typical AI style**
-- **Do not generate the following types of documents in the project:**
-  - Benchmark summary documents (BENCHMARK*.md)
-  - Implementation comparison analysis documents (IMPLEMENTATION_COMPARISON*.md)
-  - Performance analysis report documents
-  - Architecture summary documents
-  - Feature comparison documents
-  - Any documents with large amounts of emoji and formatted content
-- **If documentation is needed, only create when explicitly requested by the user, and maintain a concise and practical style**
-- **Documentation should focus on actually needed information, avoiding excessive formatting and decorative content**
-- **Any discovered AI-generated summary documents should be immediately deleted**
-
-### Allowed Documentation Types
-
-- README.md (project introduction, keep concise)
-- Technical documentation (only create when explicitly needed)
-- User manual (only create when explicitly needed)
-- API documentation (generated from code)
-- Changelog (CHANGELOG.md)
-
-These rules should serve as guiding principles when developing the RustFS project, ensuring code quality, performance, and maintainability.

AGENTS.md

@@ -1,101 +1,67 @@
-# Repository Guidelines
+# RustFS Agent Instructions (Global)
 
-This file provides guidance for AI agents and developers working with code in this repository.
+This root file keeps repository-wide rules only.
+Use the nearest subdirectory `AGENTS.md` for path-specific guidance.
 
-## Project Overview
+## Rule Precedence
 
-RustFS is a high-performance distributed object storage software built with Rust, providing S3-compatible APIs and advanced features like data lakes, AI, and big data support. It's designed as an alternative to MinIO with better performance and a more business-friendly Apache 2.0 license.
+1. System/developer instructions.
+2. This file (global defaults).
+3. The nearest `AGENTS.md` in the current path (more specific scope wins).
 
-## ⚠️ Pre-Commit Checklist (MANDATORY)
+If repo-level instructions conflict, follow the nearest file and keep behavior aligned with CI.
+
+## Communication and Language
+
+- Respond in the same language used by the requester.
+- Keep source code, comments, commit messages, and PR title/body in English.
+
+## Sources of Truth
+
+- Workspace layout and crate membership: `Cargo.toml` (`[workspace].members`)
+- Local quality commands: `Makefile` and `.config/make/`
+- CI quality gates: `.github/workflows/ci.yml`
+- PR template: `.github/pull_request_template.md`
+
+Avoid duplicating long crate lists or command matrices in instruction files.
+Reference the source files above instead.
+
+## Mandatory Before Commit
+
+Run and pass:
 
-**Before EVERY commit, you MUST run and pass ALL of the following:**
 ```bash
-cargo fmt --all --check    # Code formatting
-cargo clippy --all-targets --all-features -- -D warnings  # Lints
-cargo test --workspace --exclude e2e_test  # Unit tests
+make pre-commit
 ```
-Or simply run `make pre-commit` which covers all checks. **DO NOT commit if any check fails.**
 
-## Communication Rules
+If `make` is unavailable, run the equivalent checks defined under `.config/make/`.
+Do not commit when required checks fail.
 
-- Respond to the user in Chinese; use English in all other contexts.
-- Code and documentation must be written in English only. Chinese text is allowed solely as test data/fixtures when a case explicitly requires Chinese-language content for validation.
-- **Pull Request titles and descriptions must be written in English** to ensure consistency and accessibility for all contributors.
+## Git and PR Baseline
 
-## Project Structure
+- Use feature branches based on the latest `main`.
+- Follow Conventional Commits, with subject length <= 72 characters.
+- Keep PR title and description in English.
+- Use `.github/pull_request_template.md` and keep all section headings.
+- Use `N/A` for non-applicable template sections.
+- Include verification commands in the PR description.
+- When using `gh pr create`/`gh pr edit`, use `--body-file` instead of inline `--body` for multiline markdown.
 
-The workspace root hosts shared dependencies in `Cargo.toml`. The service binary lives under `rustfs/src/main.rs`, while reusable crates sit in `crates/` (including `ecstore`, `iam`, `kms`, `madmin`, `s3select-api`, `s3select-query`, `config`, `crypto`, `lock`, `filemeta`, `rio`, `common`, `protos`, `audit-logger`, `notify`, `obs`, `workers`, `appauth`, `ahm`, `mcp`, `signer`, `checksums`, `utils`, `zip`, `targets`, and `e2e_test`). Deployment manifests are under `deploy/`, Docker assets sit at the root, and automation lives in `scripts/`.
+## Security Baseline
 
-### Core Architecture
+- Never commit secrets, credentials, or key material.
+- Use environment variables or vault tooling for sensitive configuration.
+- For localhost-sensitive tests, verify proxy settings to avoid traffic leakage.
 
-- **Main Binary (`rustfs/`):** Entry point at `rustfs/src/main.rs`, includes admin, auth, config, server, storage, license management, and profiling modules.
-- **Key Crates:** Cargo workspace with 25+ crates supporting S3-compatible APIs, erasure coding storage, IAM, KMS, S3 Select, and observability.
-- **Build System:** Custom `build-rustfs.sh` script, multi-architecture Docker builds, Make/Just task runners, cross-compilation support.
+## Scoped Guidance in This Repository
 
-## Build, Test, and Development Commands
-
-### Quick Commands
-- `cargo check --all-targets` - Fast validation
-- `cargo build --release` or `make build` - Release build
-- `./build-rustfs.sh --dev` - Development build with debug symbols
-- `make pre-commit` - Run all quality checks (fmt, clippy, check, test)
-
-### Testing
-- `cargo test --workspace --exclude e2e_test` - Unit tests
-- `cargo test --package e2e_test` - E2E tests
-- For KMS tests: `NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1`
-
-### Cross-Platform Builds
-- `./build-rustfs.sh --platform x86_64-unknown-linux-musl` - Build for musl
-- `./build-rustfs.sh --platform aarch64-unknown-linux-gnu` - Build for ARM64
-- `make build-cross-all` - Build all supported architectures
-
-## Coding Style & Safety Requirements
-
-- **Formatting:** Follow `rustfmt.toml` (130-column width). Use `snake_case` for items, `PascalCase` for types, `SCREAMING_SNAKE_CASE` for constants.
-- **Safety:** `unsafe_code = "deny"` enforced at workspace level. Never use `unwrap()`, `expect()`, or panic-inducing code except in tests.
-- **Error Handling:** Prefer `anyhow` for applications, `thiserror` for libraries. Use proper error handling with `Result<T, E>` and `Option<T>`.
-- **Async:** Keep async code non-blocking. Offload CPU-heavy work with `tokio::task::spawn_blocking` when necessary.
-- **Language:** Code comments, function names, variable names, and all text in source files must be in English only.
-
-## Testing Guidelines
-
-Co-locate unit tests with their modules and give behavior-led names. Integration suites belong in each crate's `tests/` directory, while exhaustive end-to-end scenarios live in `crates/e2e_test/`. When fixing bugs or adding features, include regression tests that capture the new behavior.
-
-## KMS (Key Management Service)
-
-- **Implementation:** Complete with Local and Vault backends, auto-configures on startup with `--kms-enable` flag.
-- **Encryption:** Full S3-compatible server-side encryption (SSE-S3, SSE-KMS, SSE-C).
-- **Testing:** Comprehensive E2E tests in `crates/e2e_test/src/kms/`. Requires proxy bypass for local testing.
-- **Key Files:** `crates/kms/`, `rustfs/src/storage/ecfs.rs`, `rustfs/src/admin/handlers/kms*.rs`
-
-## Environment Variables
-
-- `RUSTFS_ENABLE_SCANNER` - Enable/disable background data scanner (default: true)
-- `RUSTFS_ENABLE_HEAL` - Enable/disable auto-heal functionality (default: true)
-- For KMS tests: `NO_PROXY=127.0.0.1,localhost` and clear proxy environment variables
-
-## Commit & Pull Request Guidelines
-
-Work on feature branches (e.g., `feat/...`) after syncing `main`. Follow Conventional Commits under 72 characters. Each commit must compile, format cleanly, and pass `make pre-commit`.
-
-**Pull Request Requirements:**
-- PR titles and descriptions **MUST be written in English**
-- Open PRs with a concise summary, note verification commands, link relevant issues
-- Follow the PR template format and fill in all required sections
-- Wait for reviewer approval before merging
-
-## Security & Configuration Tips
-
-Do not commit secrets or cloud credentials; prefer environment variables or vault tooling. Review IAM- and KMS-related changes with a second maintainer. Confirm proxy settings before running sensitive tests to avoid leaking traffic outside localhost.
-
-## Important Reminders
-
-- Always compile after code changes: Use `cargo build` to catch errors early
-- Don't bypass tests: All functionality must be properly tested, not worked around
-- Use proper error handling: Never use `unwrap()` or `expect()` in production code (except tests)
-- Do what has been asked; nothing more, nothing less
-- NEVER create files unless they're absolutely necessary for achieving your goal
-- ALWAYS prefer editing an existing file to creating a new one
-- NEVER proactively create documentation files (*.md) or README files unless explicitly requested
-- NEVER commit PR description files (e.g., PR_DESCRIPTION.md): These are temporary reference files for creating pull requests and should remain local only
+- `.github/AGENTS.md`
+- `crates/AGENTS.md`
+- `crates/config/AGENTS.md`
+- `crates/ecstore/AGENTS.md`
+- `crates/e2e_test/AGENTS.md`
+- `crates/iam/AGENTS.md`
+- `crates/kms/AGENTS.md`
+- `crates/policy/AGENTS.md`
+- `rustfs/src/admin/AGENTS.md`
+- `rustfs/src/storage/AGENTS.md`

CHANGELOG.md

@@ -0,0 +1,79 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Fixed
+- **Helm Ingress**: `customAnnotations` are now merged with class-specific annotations (nginx/traefik) instead of being ignored when `ingress.className` is set.
+
+### Added
+- **OpenStack Keystone Authentication Integration**: Full support for OpenStack Keystone authentication via X-Auth-Token headers
+  - Tower-based middleware (`KeystoneAuthLayer`) self-contained within `rustfs-keystone` crate
+  - Task-local storage for async-safe credential passing between middleware and auth handlers
+  - Automatic detection of Keystone credentials (access keys prefixed with `keystone:`)
+  - Role-based permission mapping (admin/reseller_admin roles grant owner permissions)
+  - Token caching for high-performance validation with configurable cache size and TTL
+  - Dual authentication support: Keystone and standard AWS Signature v4 work simultaneously
+  - Immediate 401 response for invalid tokens (no fallback to local auth)
+  - XML-formatted error responses compatible with S3 API
+  - Comprehensive integration documentation with manual testing guide
+  - **32 unit and integration tests** covering middleware, auth handlers, task-local storage, and role detection
+
+### Changed
+- **HTTP Server Stack**: Integrated `KeystoneAuthLayer` middleware from `rustfs-keystone` crate into service stack (positioned after ReadinessGateLayer)
+- **IAMAuth**: Enhanced `get_secret_key()` to return empty secret for Keystone credentials (bypasses signature validation)
+- **Auth Module**: Modified `check_key_valid()` to retrieve Keystone credentials from task-local storage and determine admin status
+
+### Technical Details
+- Middleware is self-contained in `rustfs-keystone` crate following the trusted-proxies pattern for integration-specific middleware
+- Uses `BoxBody` pattern for Hyper 1.x compatibility
+- Task-local storage provides request-scoped credential passing without modifying HTTP request/response types
+- Integration preserves existing S3 authentication flow while adding Keystone support
+- Zero breaking changes to existing functionality
+- No new top-level directories in main binary crate (middleware lives in integration crate)
+
+### Documentation
+- Updated `crates/keystone/README.md` with complete integration architecture and workflow
+- Added detailed manual testing guide with 10 test scenarios
+- Updated main `README.md` to list Keystone authentication as available feature
+- Added troubleshooting section for common integration issues
+
+### Configuration
+New environment variables:
+- `RUSTFS_KEYSTONE_ENABLE` - Enable/disable Keystone authentication (default: false)
+- `RUSTFS_KEYSTONE_AUTH_URL` - Keystone API endpoint URL
+- `RUSTFS_KEYSTONE_VERSION` - Keystone API version (v3)
+- `RUSTFS_KEYSTONE_ADMIN_USER` - Admin username for privileged operations
+- `RUSTFS_KEYSTONE_ADMIN_PASSWORD` - Admin password
+- `RUSTFS_KEYSTONE_ADMIN_PROJECT` - Admin project name
+- `RUSTFS_KEYSTONE_ADMIN_DOMAIN` - Admin domain name (default: Default)
+- `RUSTFS_KEYSTONE_CACHE_SIZE` - Token cache size (default: 10000)
+- `RUSTFS_KEYSTONE_CACHE_TTL` - Token cache TTL in seconds (default: 300)
+- `RUSTFS_KEYSTONE_VERIFY_SSL` - Verify SSL certificates (default: true)
+
+### Files Modified
+- `crates/keystone/src/middleware.rs` - Created Keystone authentication middleware (self-contained in keystone crate)
+- `crates/keystone/src/lib.rs` - Exported middleware module and KEYSTONE_CREDENTIALS
+- `crates/keystone/Cargo.toml` - Added Tower/HTTP dependencies for middleware functionality
+- `rustfs/src/server/http.rs` - Integrated KeystoneAuthLayer from rustfs-keystone crate
+- `rustfs/src/auth.rs` - Enhanced IAMAuth and check_key_valid for Keystone support, imported KEYSTONE_CREDENTIALS from rustfs-keystone
+- `crates/keystone/README.md` - Comprehensive integration documentation
+- `README.md` - Added Keystone as available feature
+
+### Testing
+- 16 unit tests in rustfs-keystone crate (config, auth, middleware, identity)
+- 10 integration tests in rustfs-keystone crate (task-local storage, middleware layer, scope isolation)
+- 6 auth unit tests in rustfs crate (role detection, task-local storage, Keystone credential handling)
+- **Total: 32 tests** passing with zero compilation errors
+- Manual testing guide provided for end-to-end validation
+- All tests passing with `cargo test --all --exclude e2e_test`
+
+---
+
+## Previous Releases
+
+See [GitHub Releases](https://github.com/rustfs/rustfs/releases) for previous version history.

CLA.md

@@ -1,85 +1,60 @@
-RustFS Individual Contributor License Agreement
+# RustFS Individual Contributor License Agreement
 
-Thank you for your interest in contributing documentation and related software code to a project hosted or managed by
-RustFS. In order to clarify the intellectual property license granted with Contributions from any person or entity,
-RustFS must have a Contributor License Agreement ("CLA") on file that has been signed by each Contributor, indicating
-agreement to the license terms below. This version of the Contributor License Agreement allows an individual to submit
-Contributions to the applicable project. If you are making a submission on behalf of a legal entity, then you should
-sign the separate Corporate Contributor License Agreement.
+Thank you for your interest in contributing to RustFS. In order to clarify the intellectual property license granted with Contributions from any person or entity, RustFS, Inc. ("RustFS") must have a Contributor License Agreement ("CLA") on file that has been signed by each Contributor, indicating agreement to the license terms below. This license is for your protection as a Contributor as well as the protection of RustFS and its users; it does not change your rights to use your own Contributions for any other purpose.
 
-You accept and agree to the following terms and conditions for Your present and future Contributions submitted to
-RustFS. You hereby irrevocably assign and transfer to RustFS all right, title, and interest in and to Your
-Contributions, including all copyrights and other intellectual property rights therein.
+You accept and agree to the following terms and conditions for Your present and future Contributions submitted to RustFS. Except for the license granted herein to RustFS and recipients of software distributed by RustFS, You reserve all right, title, and interest in and to Your Contributions.
 
-Definitions
+**You understand and agree that You will not receive any royalty or other compensation for Your Contributions.**
 
-“You” (or “Your”) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this
-Agreement with RustFS. For legal entities, the entity making a Contribution and all other entities that control, are
-controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes
-of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such
-entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares,
-or (iii) beneficial ownership of such entity.
+## 1. Definitions
 
-“Contribution” shall mean any original work of authorship, including any modifications or additions to an existing work,
-that is intentionally submitted by You to RustFS for inclusion in, or documentation of, any of the products or projects
-owned or managed by RustFS (the "Work"), including without limitation any Work described in Schedule A. For the purposes
-of this definition, "submitted" means any form of electronic or written communication sent to RustFS or its
-representatives, including but not limited to communication on electronic mailing lists, source code control systems,
-and issue tracking systems that are managed by, or on behalf of, RustFS for the purpose of discussing and improving the
-Work.
+*   **"You"** (or **"Your"**) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with RustFS. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
 
-Assignment of Copyright
+*   **"Contribution"** shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to RustFS for inclusion in, or documentation of, any of the products or projects owned or managed by RustFS (the "Work"). For the purposes of this definition, "submitted" means any form of electronic or written communication sent to RustFS or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, RustFS for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution."
 
-Subject to the terms and conditions of this Agreement, You hereby irrevocably assign and transfer to RustFS all right,
-title, and interest in and to Your Contributions, including all copyrights and other intellectual property rights
-therein, for the entire term of such rights, including all renewals and extensions. You agree to execute all documents
-and take all actions as may be reasonably necessary to vest in RustFS the ownership of Your Contributions and to assist
-RustFS in perfecting, maintaining, and enforcing its rights in Your Contributions.
+## 2. Grant of Copyright License
 
-Grant of Patent License
+Subject to the terms and conditions of this Agreement, You hereby grant to RustFS and to recipients of software distributed by RustFS a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works under any license, including proprietary or commercial licenses and open-source licenses, at RustFS's sole discretion.
 
-Subject to the terms and conditions of this Agreement, You hereby grant to RustFS and to recipients of documentation and
-software distributed by RustFS a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as
-stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the
-Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your
-Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was
-submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or
-counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes
-direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for
-that Contribution or Work shall terminate as of the date such litigation is filed.
+## 3. Grant of Patent License
 
-You represent that you are legally entitled to grant the above assignment and license.
+Subject to the terms and conditions of this Agreement, You hereby grant to RustFS and to recipients of software distributed by RustFS a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.
 
-You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of
-others). You represent that Your Contribution submissions include complete details of any third-party license or other
-restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which
-are associated with any part of Your Contributions.
+## 4. Representations
 
-You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You
-may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You
-provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
-including, without limitation, any warranties or conditions of TITLE, NON- INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR
-A PARTICULAR PURPOSE.
+You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to RustFS, or that your employer has executed a separate Corporate CLA with RustFS.
 
-Should You wish to submit work that is not Your original creation, You may submit it to RustFS separately from any
-Contribution, identifying the complete details of its source and of any license or other restriction (including, but not
-limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously
-marking the work as "Submitted on behalf of a third-party: [named here]”.
+You represent that each of Your Contributions is Your original creation. You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which are associated with any part of Your Contributions.
 
-You agree to notify RustFS of any facts or circumstances of which you become aware that would make these representations
-inaccurate in any respect.
+## 5. Support and Warranty
 
-Modification of CLA
+You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of **TITLE**, **NON-INFRINGEMENT**, **MERCHANTABILITY**, or **FITNESS FOR A PARTICULAR PURPOSE**.
 
-RustFS reserves the right to update or modify this CLA in the future. Any updates or modifications to this CLA shall
-apply only to Contributions made after the effective date of the revised CLA. Contributions made prior to the update
-shall remain governed by the version of the CLA that was in effect at the time of submission. It is not necessary for
-all Contributors to re-sign the CLA when the CLA is updated or modified.
+## 6. Third-Party Work
 
-Governing Law and Dispute Resolution
+Should You wish to submit work that is not Your original creation, You may submit it to RustFS separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as "Submitted on behalf of a third-party: [named here]".
 
-This Agreement will be governed by and construed in accordance with the laws of the People's Republic of China excluding
-that body of laws known as conflict of laws. The parties expressly agree that the United Nations Convention on Contracts
-for the International Sale of Goods will not apply. Any legal action or proceeding arising under this Agreement will be
-brought exclusively in the courts located in Beijing, China, and the parties hereby irrevocably consent to the personal
-jurisdiction and venue therein.
+## 7. Governing Law and Jurisdiction
+
+This Agreement shall be governed by and construed in accordance with the laws of the State of Delaware, United States of America, without regard to its conflict of laws principles. The parties expressly agree that the United Nations Convention on Contracts for the International Sale of Goods will not apply. Any legal action or proceeding arising under or in connection with this Agreement shall be brought exclusively in the state or federal courts located in the State of Delaware, United States of America, and the parties hereby irrevocably consent to the personal jurisdiction and venue therein.
+
+## 8. Severability
+
+If any provision of this Agreement is found to be invalid or unenforceable, the remaining provisions will continue in full force and effect.
+
+## 9. Entire Agreement and Assignment
+
+This Agreement constitutes the entire agreement between the parties with respect to the subject matter hereof and supersedes all prior agreements, understandings, negotiations, and discussions, whether written or oral. RustFS may assign its rights and obligations under this Agreement to any third party. You may not assign Your rights or obligations under this Agreement without the prior written consent of RustFS.
+
+---
+
+**Please read the terms of this Agreement carefully. By submitting a Contribution to RustFS, You agree to be bound by the terms and conditions of this Agreement.**
+
+| | |
+|---|---|
+| **Full Name** | __________________________________ |
+| **GitHub Username** | __________________________________ |
+| **Email Address** | __________________________________ |
+| **Date** | __________________________________ |
+
+*(Electronic signature or acknowledgement via GitHub commit/Pull Request constitutes valid acceptance of this Agreement).*

Cargo.toml

@@ -27,6 +27,7 @@ members = [
     "crates/filemeta", # File metadata management
     "crates/heal", # Erasure set and object healing
     "crates/iam", # Identity and Access Management
+    "crates/keystone", # OpenStack Keystone integration
     "crates/kms", # Key Management Service
     "crates/lock", # Distributed locking implementation
     "crates/madmin", # Management dashboard and admin API interface
@@ -38,6 +39,7 @@ members = [
     "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
     "crates/protos", # Protocol buffer definitions
     "crates/rio", # Rust I/O utilities and abstractions
+    "crates/s3-common", # Common utilities and data structures for S3 compatibility
     "crates/s3select-api", # S3 Select API interface
     "crates/s3select-query", # S3 Select query engine
     "crates/scanner", # Scanner for data integrity checks and health monitoring
@@ -48,7 +50,7 @@ members = [
     "crates/workers", # Worker thread pools and task scheduling
     "crates/zip", # ZIP file handling and compression
 ]
-resolver = "2"
+resolver = "3"
 
 [workspace.package]
 edition = "2024"
@@ -82,6 +84,7 @@ rustfs-crypto = { path = "crates/crypto", version = "0.0.5" }
 rustfs-ecstore = { path = "crates/ecstore", version = "0.0.5" }
 rustfs-filemeta = { path = "crates/filemeta", version = "0.0.5" }
 rustfs-iam = { path = "crates/iam", version = "0.0.5" }
+rustfs-keystone = { path = "crates/keystone", version = "0.0.5" }
 rustfs-kms = { path = "crates/kms", version = "0.0.5" }
 rustfs-lock = { path = "crates/lock", version = "0.0.5" }
 rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
@@ -92,6 +95,7 @@ rustfs-obs = { path = "crates/obs", version = "0.0.5" }
 rustfs-policy = { path = "crates/policy", version = "0.0.5" }
 rustfs-protos = { path = "crates/protos", version = "0.0.5" }
 rustfs-rio = { path = "crates/rio", version = "0.0.5" }
+rustfs-s3-common = { path = "crates/s3-common", version = "0.0.5" }
 rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
 rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.5" }
 rustfs-scanner = { path = "crates/scanner", version = "0.0.5" }
@@ -105,14 +109,13 @@ rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
-async-compression = { version = "0.4.39" }
+async-compression = { version = "0.4.41" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
 axum = "0.8.8"
-axum-server = { version = "0.8.0", features = ["tls-rustls"], default-features = false }
-futures = "0.3.31"
-futures-core = "0.3.31"
-futures-util = "0.3.31"
+futures = "0.3.32"
+futures-core = "0.3.32"
+futures-util = "0.3.32"
 pollster = "0.4.0"
 hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
 hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
@@ -121,15 +124,15 @@ http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
 reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
-socket2 = { version = "0.6.2", features = ["all"] }
-tokio = { version = "1.49.0", features = ["fs", "rt-multi-thread"] }
+socket2 = { version = "0.6.3", features = ["all"] }
+tokio = { version = "1.50.0", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
 tokio-util = { version = "0.7.18", features = ["io", "compat"] }
-tonic = { version = "0.14.3", features = ["gzip"] }
-tonic-prost = { version = "0.14.3" }
-tonic-prost-build = { version = "0.14.3" }
+tonic = { version = "0.14.5", features = ["gzip"] }
+tonic-prost = { version = "0.14.5" }
+tonic-prost-build = { version = "0.14.5" }
 tower = { version = "0.5.3", features = ["timeout"] }
 tower-http = { version = "0.6.8", features = ["cors"] }
 
@@ -140,8 +143,8 @@ byteorder = "1.5.0"
 flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.3"
-quick-xml = "0.39.0"
-rmcp = { version = "0.15.0" }
+quick-xml = "0.39.2"
+rmcp = { version = "1.2.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
@@ -157,10 +160,10 @@ chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
 hmac = { version = "0.13.0-rc.5" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
+openidconnect = { version = "4.0", default-features = false }
 pbkdf2 = "0.13.0-rc.9"
-rsa = { version = "0.10.0-rc.15" }
-rustls = { version = "0.23.36", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
-rustls-pemfile = "2.2.0"
+rsa = { version = "0.10.0-rc.17" }
+rustls = { version = "0.23.37", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types = "1.14.0"
 sha1 = "0.11.0-rc.5"
 sha2 = "0.11.0-rc.5"
@@ -168,46 +171,47 @@ subtle = "2.6"
 zeroize = { version = "1.8.2", features = ["derive"] }
 
 # Time and Date
-chrono = { version = "0.4.43", features = ["serde"] }
+chrono = { version = "0.4.44", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.19", features = ["serde"] }
+jiff = { version = "0.2.23", features = ["serde"] }
 time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
 
 # Utilities and Tools
-anyhow = "1.0.101"
-arc-swap = "1.8.1"
+anyhow = "1.0.102"
+arc-swap = "1.8.2"
 astral-tokio-tar = "0.5.6"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.13" }
-aws-credential-types = { version = "1.2.11" }
-aws-sdk-s3 = { version = "1.122.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
-aws-smithy-types = { version = "1.4.4" }
+aws-config = { version = "1.8.15" }
+aws-credential-types = { version = "1.2.14" }
+aws-sdk-s3 = { version = "1.126.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
+aws-smithy-types = { version = "1.4.6" }
 backtrace = "0.3.76"
 base64 = "0.22.1"
 base64-simd = "0.8.0"
 brotli = "8.0.2"
 cfg-if = "1.0.4"
-clap = { version = "4.5.57", features = ["derive", "env"] }
+clap = { version = "4.6.0", features = ["derive", "env"] }
 const-str = { version = "1.1.0", features = ["std", "proc"] }
 convert_case = "0.11.0"
 criterion = { version = "0.8", features = ["html_reports"] }
 crossbeam-queue = "0.3.12"
-datafusion = "52.1.0"
+datafusion = "52.3.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
 flate2 = "1.1.9"
-flexi_logger = { version = "0.31.8", features = ["trc", "dont_minimize_extra_stacks", "compress", "kv", "json"] }
 glob = "0.3.3"
-google-cloud-storage = "1.7.0"
-google-cloud-auth = "1.5.0"
+google-cloud-storage = "1.9.0"
+google-cloud-auth = "1.7.0"
 hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
+hex = "0.4.3"
 hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.181"
+libc = "0.2.183"
 libsystemd = "0.7.2"
 local-ip-address = "0.6.10"
 lz4 = "1.28.1"
@@ -215,55 +219,56 @@ matchit = "0.9.1"
 md-5 = "0.11.0-rc.5"
 md5 = "0.8.0"
 mime_guess = "2.0.5"
-moka = { version = "0.12.13", features = ["future"] }
+moka = { version = "0.12.14", features = ["future"] }
 netif = "0.1.6"
-nu-ansi-term = "0.50.3"
 num_cpus = { version = "1.17.0" }
-nvml-wrapper = "0.11.0"
+nvml-wrapper = "0.12.0"
 object_store = "0.12.5"
 parking_lot = "0.12.5"
 path-absolutize = "3.1.1"
 path-clean = "1.0.1"
-pin-project-lite = "0.2.16"
+percent-encoding = "2.3.2"
+pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
 rand = { version = "0.10.0", features = ["serde"] }
+ratelimit = "0.10.0"
 rayon = "1.11.0"
 reed-solomon-simd = { version = "3.1.0" }
 regex = { version = "1.12.3" }
 rumqttc = { version = "0.25.1" }
-rustix = { version = "1.1.3", features = ["fs"] }
+rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.1" }
-s3s = { version = "0.13.0-alpha.3", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", rev = "7b8e22b49bc76839e6085a4ddfa553323f1b8049" }
-serial_test = "3.3.1"
-shadow-rs = { version = "1.7.0", default-features = false }
+s3s = { git = "https://github.com/s3s-project/s3s", rev = "c2dc7b16535659904d4efff52c558fc039be1ef3", features = ["minio"] }
+serial_test = "3.4.0"
+shadow-rs = { version = "1.7.1", default-features = false }
 siphasher = "1.0.2"
 smallvec = { version = "1.15.1", features = ["serde"] }
 smartstring = "1.0.1"
-snafu = "0.8.9"
+snafu = "0.9.0"
 snap = "1.1.1"
 starshard = { version = "1.1.0", features = ["rayon", "async", "serde"] }
-strum = { version = "0.27.2", features = ["derive"] }
-sysinfo = "0.38.1"
+strum = { version = "0.28.0", features = ["derive"] }
+sysinfo = "0.38.4"
 temp-env = "0.3.6"
-tempfile = "3.25.0"
+tempfile = "3.27.0"
 test-case = "3.3.1"
 thiserror = "2.0.18"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.4"
 tracing-error = "0.2.1"
 tracing-opentelemetry = "0.32.1"
-tracing-subscriber = { version = "0.3.22", features = ["env-filter", "time"] }
+tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.20.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.22.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
 vaultrs = { version = "0.7.4" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "7.4.0"
+zip = "8.2.0"
 zstd = "0.13.3"
 
 # Observability and Metrics
@@ -274,12 +279,17 @@ opentelemetry-otlp = { version = "0.31.0", features = ["gzip-http", "reqwest-rus
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }
+pyroscope = { version = "2.0.0", features = ["backend-pprof-rs"] }
 
 # FTP and SFTP
-libunftp = { version = "0.22.0", features = ["experimental"] }
-suppaftp = { version = "8.0.1", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
+libunftp = { version = "0.23.0", features = ["experimental"] }
+unftp-core = "0.1.0"
+suppaftp = { version = "8.0.2", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.7"
 
+# WebDAV
+dav-server = "0.11.0"
+
 # Performance Analysis and Memory Profiling
 mimalloc = "0.1"
 # Use tikv-jemallocator as memory allocator and enable performance analysis
@@ -289,7 +299,9 @@ tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats", "profilin
 # Used to generate pprof-compatible memory profiling data and support symbolization and flame graphs
 jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
 # Used to generate CPU performance analysis data and flame diagrams
-pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] }
+# pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] }
+# Pyroscope uses a patched pprof, until they merge back upstream, replace all references. Otherwise, two pprof libs with symbol collision.
+pprof = { package = "pprof-pyroscope-fork", version = "0.1500.3", features = ["flamegraph", "protobuf-codec"] }
 
 [workspace.metadata.cargo-shear]
 ignored = ["rustfs", "rustfs-mcp"]

Dockerfile

@@ -1,3 +1,17 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 FROM alpine:3.23 AS build
 
 ARG TARGETARCH

Dockerfile.glibc

@@ -1,3 +1,17 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 FROM ubuntu:24.04 AS build
 
 ARG TARGETARCH

Dockerfile.source

@@ -1,4 +1,18 @@
 # syntax=docker/dockerfile:1.6
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Multi-stage Dockerfile for RustFS - LOCAL DEVELOPMENT ONLY
 #
 # IMPORTANT: This Dockerfile builds RustFS from source for local development and testing.

README.md

@@ -1,4 +1,4 @@
-[![RustFS](https://github.com/user-attachments/assets/1b5afcd6-a2c3-47ff-8bc3-ce882b0ddca7)](https://rustfs.com)
+[![RustFS](https://repository-images.githubusercontent.com/722597620/0fa936a2-8164-4f53-867f-def4beb64b21)](https://rustfs.com)
 
 <p align="center">RustFS is a high-performance, distributed object storage system built in Rust.</p>
 
@@ -42,6 +42,8 @@ Unlike other storage systems, RustFS is released under the permissible Apache 2.
 - **High Performance**: Built with Rust to ensure maximum speed and resource efficiency.
 - **Distributed Architecture**: Scalable and fault-tolerant design suitable for large-scale deployments.
 - **S3 Compatibility**: Seamless integration with existing S3-compatible applications and tools.
+- **OpenStack Swift API**: Native support for Swift protocol with Keystone authentication.
+- **OpenStack Keystone Integration**: Native support for OpenStack Keystone authentication with X-Auth-Token headers.
 - **Data Lake Support**: Optimized for high-throughput big data and AI workloads.
 - **Open Source**: Licensed under Apache 2.0, encouraging unrestricted community contributions and commercial usage.
 - **User-Friendly**: Designed with simplicity in mind for easy deployment and management.
@@ -54,6 +56,8 @@ Unlike other storage systems, RustFS is released under the permissible Apache 2.
 | **Logging**             | ✅ Available | **Lifecycle Management** | 🚧 Under Testing |
 | **Event Notifications** | ✅ Available | **Distributed Mode**     | 🚧 Under Testing |
 | **K8s Helm Charts**     | ✅ Available | **RustFS KMS**           | 🚧 Under Testing |
+| **Keystone Auth**       | ✅ Available | **Multi-Tenancy**        | ✅ Available     |
+| **Swift API**           | ✅ Available | **Swift Metadata Ops**   | 🚧 Partial       |
 
 ## RustFS vs MinIO Performance
 

README_ZH.md

@@ -100,7 +100,7 @@ curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
 
 ### 2\. Docker 快速启动 (选项 2)
 
-RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用 Docker 的 `-v` 参数挂载宿主机目录，请务必确保宿主机目录的所有者已更改为 `1000`，否则会遇到权限拒绝错误。
+RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用 Docker 的 `-v` 参数挂载宿主机目录，请务必确保宿主机目录的所有者已更改为 `10001`，否则会遇到权限拒绝错误。
 
 ```bash
  # 创建数据和日志目录

_typos.toml

@@ -42,6 +42,8 @@ GAE = "GAE"
 # s3-tests original test names (cannot be changed)
 nonexisted = "nonexisted"
 consts = "consts"
+# Swift API - company/product names
+Hashi = "Hashi"  # HashiCorp
 
 [files]
 extend-exclude = []

crates/AGENTS.md

@@ -0,0 +1,20 @@
+# Crates Instructions
+
+Applies to all paths under `crates/`.
+
+## Library Design
+
+- Treat crate code as reusable library code by default.
+- Prefer `thiserror` for library-facing error types.
+- Do not use `unwrap()`, `expect()`, or panic-driven control flow outside tests.
+
+## Testing
+
+- Keep unit tests close to the module they test.
+- Keep integration tests under each crate's `tests/` directory.
+- Add regression tests for bug fixes and behavior changes.
+
+## Async and Performance
+
+- Keep async paths non-blocking.
+- Move CPU-heavy operations out of async hot paths with `tokio::task::spawn_blocking` when appropriate.

crates/audit/Cargo.toml

@@ -29,6 +29,7 @@ categories = ["web-programming", "development-tools", "asynchronous", "api-bindi
 rustfs-targets = { workspace = true }
 rustfs-config = { workspace = true, features = ["audit", "constants"] }
 rustfs-ecstore = { workspace = true }
+rustfs-s3-common = { workspace = true }
 async-trait = { workspace = true }
 chrono = { workspace = true }
 const-str = { workspace = true }

crates/audit/src/entity.rs

@@ -14,7 +14,7 @@
 
 use chrono::{DateTime, Utc};
 use hashbrown::HashMap;
-use rustfs_targets::EventName;
+use rustfs_s3_common::EventName;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 

crates/audit/src/factory.rs

@@ -19,8 +19,9 @@ use rumqttc::QoS;
 use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
 use rustfs_config::{
     AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT,
-    WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
+    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
+    WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
+    WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
@@ -75,6 +76,11 @@ impl TargetFactory for WebhookTargetFactory {
                 .unwrap_or(DEFAULT_LIMIT),
             client_cert: config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(),
             client_key: config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(),
+            client_ca: config.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
+            skip_tls_verify: config
+                .lookup(WEBHOOK_SKIP_TLS_VERIFY)
+                .and_then(|v| v.parse::<bool>().ok())
+                .unwrap_or(RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT),
             target_type: rustfs_targets::target::TargetType::AuditLog,
         };
 

crates/common/Cargo.toml

@@ -32,6 +32,7 @@ tokio = { workspace = true }
 tonic = { workspace = true }
 uuid = { workspace = true }
 chrono = { workspace = true }
+metrics = { workspace = true }
 rustfs-madmin = { workspace = true }
 rustfs-filemeta = { workspace = true }
 serde = { workspace = true }
@@ -39,4 +40,4 @@ path-clean = { workspace = true }
 rmp-serde = { workspace = true }
 async-trait = { workspace = true }
 s3s = { workspace = true }
-tracing = { workspace = true }
+tracing = { workspace = true }

crates/common/src/data_usage.rs

@@ -458,10 +458,7 @@ impl DataUsageEntry {
         self.size += other.size;
 
         if let Some(o_rep) = &other.replication_stats {
-            if self.replication_stats.is_none() {
-                self.replication_stats = Some(ReplicationAllStats::default());
-            }
-            let s_rep = self.replication_stats.as_mut().unwrap();
+            let s_rep = self.replication_stats.get_or_insert_with(ReplicationAllStats::default);
             s_rep.targets.clear();
             s_rep.replica_size += o_rep.replica_size;
             s_rep.replica_count += o_rep.replica_count;
@@ -586,7 +583,7 @@ impl DataUsageCache {
                     return Some(root);
                 }
                 let mut flat = self.flatten(&root);
-                if flat.replication_stats.is_some() && flat.replication_stats.as_ref().unwrap().empty() {
+                if flat.replication_stats.as_ref().is_some_and(|stats| stats.empty()) {
                     flat.replication_stats = None;
                 }
                 Some(flat)
@@ -679,7 +676,9 @@ impl DataUsageCache {
         leaves.sort_by(|a, b| a.objects.cmp(&b.objects));
 
         while remove > 0 && !leaves.is_empty() {
-            let e = leaves.first().unwrap();
+            let Some(e) = leaves.first() else {
+                break;
+            };
             let candidate = e.path.clone();
             if candidate == *path && !compact_self {
                 break;
@@ -703,12 +702,9 @@ impl DataUsageCache {
     }
 
     pub fn total_children_rec(&self, path: &str) -> usize {
-        let root = self.find(path);
-
-        if root.is_none() {
+        let Some(root) = self.find(path) else {
             return 0;
-        }
-        let root = root.unwrap();
+        };
         if root.children.is_empty() {
             return 0;
         }
@@ -721,31 +717,36 @@ impl DataUsageCache {
     }
 
     pub fn merge(&mut self, o: &DataUsageCache) {
-        let mut existing_root = self.root();
-        let other_root = o.root();
-        if existing_root.is_none() && other_root.is_none() {
-            return;
-        }
-        if other_root.is_none() {
-            return;
-        }
-        if existing_root.is_none() {
+        let Some(mut existing_root) = self.root() else {
+            if o.root().is_none() {
+                return;
+            }
             *self = o.clone();
             return;
-        }
-        if o.info.last_update.gt(&self.info.last_update) {
+        };
+
+        let Some(other_root) = o.root() else {
+            return;
+        };
+
+        if o.info.last_update > self.info.last_update {
             self.info.last_update = o.info.last_update;
         }
 
-        existing_root.as_mut().unwrap().merge(other_root.as_ref().unwrap());
-        self.cache.insert(hash_path(&self.info.name).key(), existing_root.unwrap());
-        let e_hash = self.root_hash();
-        for key in other_root.as_ref().unwrap().children.iter() {
-            let entry = &o.cache[key];
+        existing_root.merge(&other_root);
+        self.cache.insert(hash_path(&self.info.name).key(), existing_root);
+
+        let root_hash = self.root_hash();
+        for key in other_root.children.iter() {
+            let Some(entry) = o.cache.get(key) else {
+                continue;
+            };
             let flat = o.flatten(entry);
-            let mut existing = self.cache[key].clone();
-            existing.merge(&flat);
-            self.replace_hashed(&DataUsageHash(key.clone()), &Some(e_hash.clone()), &existing);
+            if let Some(existing) = self.cache.get_mut(key) {
+                existing.merge(&flat);
+            } else {
+                self.replace_hashed(&DataUsageHash(key.clone()), &Some(root_hash.clone()), &flat);
+            }
         }
     }
 
@@ -1141,10 +1142,12 @@ impl DataUsageInfo {
         self.buckets_count = self.buckets_usage.len() as u64;
 
         // Update last update time
-        if let Some(other_update) = other.last_update
-            && (self.last_update.is_none() || other_update > self.last_update.unwrap())
-        {
-            self.last_update = Some(other_update);
+        if let Some(other_update) = other.last_update {
+            match self.last_update {
+                None => self.last_update = Some(other_update),
+                Some(self_update) if other_update > self_update => self.last_update = Some(other_update),
+                _ => {}
+            }
         }
     }
 }
@@ -1285,4 +1288,59 @@ mod tests {
         assert_eq!(summary1.total_size, 300);
         assert_eq!(summary1.versions, 15);
     }
+
+    #[test]
+    fn test_data_usage_cache_merge_adds_missing_child() {
+        let mut base = DataUsageCache::default();
+        base.info.name = "bucket".to_string();
+        base.replace("bucket", "", DataUsageEntry::default());
+
+        let mut other = DataUsageCache::default();
+        other.info.name = "bucket".to_string();
+        let child = DataUsageEntry {
+            size: 42,
+            ..Default::default()
+        };
+        other.replace("bucket/child", "bucket", child);
+
+        base.merge(&other);
+
+        let root = base.find("bucket").expect("root bucket should exist");
+        assert_eq!(root.size, 0);
+        let child_entry = base.find("bucket/child").expect("merged child should be added");
+        assert_eq!(child_entry.size, 42);
+    }
+
+    #[test]
+    fn test_data_usage_cache_merge_accumulates_existing_child() {
+        let mut base = DataUsageCache::default();
+        base.info.name = "bucket".to_string();
+        base.replace(
+            "bucket/child",
+            "bucket",
+            DataUsageEntry {
+                size: 10,
+                objects: 1,
+                ..Default::default()
+            },
+        );
+
+        let mut other = DataUsageCache::default();
+        other.info.name = "bucket".to_string();
+        other.replace(
+            "bucket/child",
+            "bucket",
+            DataUsageEntry {
+                size: 20,
+                objects: 2,
+                ..Default::default()
+            },
+        );
+
+        base.merge(&other);
+
+        let child_entry = base.find("bucket/child").expect("child should remain after merge");
+        assert_eq!(child_entry.size, 30);
+        assert_eq!(child_entry.objects, 3);
+    }
 }

crates/common/src/globals.rs

@@ -38,6 +38,14 @@ pub async fn set_global_local_node_name(name: &str) {
     *GLOBAL_LOCAL_NODE_NAME.write().await = name.to_string();
 }
 
+/// Get the global local node name.
+///
+/// # Returns
+/// * `String` - The local node name.
+pub async fn get_global_local_node_name() -> String {
+    GLOBAL_LOCAL_NODE_NAME.read().await.clone()
+}
+
 /// Set the global RustFS initialization time to the current UTC time.
 pub async fn set_global_init_time_now() {
     let now = Utc::now();

crates/common/src/heal_channel.rs

@@ -296,12 +296,13 @@ type HealResponseSender = broadcast::Sender<HealChannelResponse>;
 static GLOBAL_HEAL_RESPONSE_SENDER: OnceLock<HealResponseSender> = OnceLock::new();
 
 /// Initialize global heal channel
-pub fn init_heal_channel() -> HealChannelReceiver {
+pub fn init_heal_channel() -> Result<HealChannelReceiver, &'static str> {
     let (tx, rx) = mpsc::unbounded_channel();
-    GLOBAL_HEAL_CHANNEL_SENDER
-        .set(tx)
-        .expect("Heal channel sender already initialized");
-    rx
+    if GLOBAL_HEAL_CHANNEL_SENDER.set(tx).is_ok() {
+        Ok(rx)
+    } else {
+        Err("Heal channel sender already initialized")
+    }
 }
 
 /// Get global heal channel sender

crates/common/src/last_minute.rs

@@ -155,10 +155,7 @@ impl LastMinuteLatency {
     }
 
     pub fn add(&mut self, t: &Duration) {
-        let sec = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .expect("Time went backwards")
-            .as_secs();
+        let sec = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
         self.forward_to(sec);
         let win_idx = sec % 60;
         self.totals[win_idx as usize].add(t);
@@ -174,10 +171,7 @@ impl LastMinuteLatency {
 
     pub fn get_total(&mut self) -> AccElem {
         let mut res = AccElem::default();
-        let sec = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .expect("Time went backwards")
-            .as_secs();
+        let sec = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
         self.forward_to(sec);
         for elem in self.totals.iter() {
             res.merge(elem);

crates/common/src/metrics.rs

@@ -14,7 +14,8 @@
 
 use crate::last_minute::{AccElem, LastMinuteLatency};
 use chrono::{DateTime, Utc};
-use rustfs_madmin::metrics::ScannerMetrics as M_ScannerMetrics;
+use rustfs_madmin::metrics::{ScannerMetrics as M_ScannerMetrics, TimedAction};
+use serde::{Deserialize, Serialize};
 use std::{
     collections::HashMap,
     fmt::Display,
@@ -64,6 +65,37 @@ impl IlmAction {
             || *self == Self::DeleteAllVersionsAction
             || *self == Self::DelMarkerDeleteAllVersionsAction
     }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::NoneAction => "none",
+            Self::DeleteAction => "delete",
+            Self::DeleteVersionAction => "delete_version",
+            Self::TransitionAction => "transition",
+            Self::TransitionVersionAction => "transition_version",
+            Self::DeleteRestoredAction => "delete_restored",
+            Self::DeleteRestoredVersionAction => "delete_restored_version",
+            Self::DeleteAllVersionsAction => "delete_all_versions",
+            Self::DelMarkerDeleteAllVersionsAction => "del_marker_delete_all_versions",
+            Self::ActionCount => "action_count",
+        }
+    }
+
+    pub fn from_index(i: usize) -> Option<Self> {
+        match i {
+            0 => Some(Self::NoneAction),
+            1 => Some(Self::DeleteAction),
+            2 => Some(Self::DeleteVersionAction),
+            3 => Some(Self::TransitionAction),
+            4 => Some(Self::TransitionVersionAction),
+            5 => Some(Self::DeleteRestoredAction),
+            6 => Some(Self::DeleteRestoredVersionAction),
+            7 => Some(Self::DeleteAllVersionsAction),
+            8 => Some(Self::DelMarkerDeleteAllVersionsAction),
+            9 => Some(Self::ActionCount),
+            _ => None,
+        }
+    }
 }
 
 impl Display for IlmAction {
@@ -272,14 +304,76 @@ pub struct Metrics {
     cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
 }
 
-// This is a placeholder. We'll need to define this struct.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct CurrentCycle {
     pub current: u64,
+    pub next: u64,
     pub cycle_completed: Vec<DateTime<Utc>>,
     pub started: DateTime<Utc>,
 }
 
+impl CurrentCycle {
+    pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        *self = rmp_serde::from_slice(buf)?;
+        Ok(())
+    }
+
+    pub fn marshal(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
+        Ok(rmp_serde::to_vec(self)?)
+    }
+}
+
+/// OTEL metric name constants for scanner metrics
+const OTEL_SCANNER_OBJECTS_SCANNED: &str = "rustfs_scanner_objects_scanned_total";
+const OTEL_SCANNER_DIRECTORIES_SCANNED: &str = "rustfs_scanner_directories_scanned_total";
+const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
+const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
+const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
+const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
+
+/// Emit an OTEL counter increment for the given scanner metric.
+/// ScanCycle and ScanBucketDrive are handled by dedicated emit functions with labels.
+fn emit_otel_counter(metric: usize, count: u64) {
+    match Metric::from_index(metric) {
+        Some(Metric::ScanObject) => {
+            metrics::counter!(OTEL_SCANNER_OBJECTS_SCANNED).increment(count);
+        }
+        Some(Metric::ScanFolder) => {
+            metrics::counter!(OTEL_SCANNER_DIRECTORIES_SCANNED).increment(count);
+        }
+        _ => {}
+    }
+}
+
+/// Emit OTel metrics for a completed scan cycle.
+/// Counter with result label + gauge for last successful cycle duration.
+pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
+    let result = if success { "success" } else { "error" };
+    metrics::counter!(OTEL_SCANNER_CYCLES, "result" => result).increment(1);
+    if success {
+        metrics::gauge!(OTEL_SCANNER_CYCLE_DURATION_SECONDS).set(duration.as_secs_f64());
+    }
+}
+
+/// Emit OTel metrics for a completed bucket-drive scan.
+/// Counter with result/bucket/disk labels + histogram for duration.
+pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
+    let result = if success { "success" } else { "error" };
+    metrics::counter!(
+        OTEL_SCANNER_BUCKETS_SCANNED,
+        "result" => result,
+        "bucket" => bucket.to_owned(),
+        "disk" => disk.to_owned()
+    )
+    .increment(1);
+    metrics::histogram!(
+        OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS,
+        "bucket" => bucket.to_owned(),
+        "disk" => disk.to_owned()
+    )
+    .record(duration.as_secs_f64());
+}
+
 impl Metrics {
     pub fn new() -> Self {
         let operations = (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect();
@@ -307,6 +401,7 @@ impl Metrics {
 
             // Update operation count
             global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric, 1);
 
             // Update latency for realtime metrics (spawn async task for this)
             if (metric) < Metric::LastRealtime as usize {
@@ -332,6 +427,7 @@ impl Metrics {
 
             // Update operation count
             global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric, 1);
 
             // Update latency for realtime metrics with size (spawn async task)
             if (metric) < Metric::LastRealtime as usize {
@@ -352,6 +448,7 @@ impl Metrics {
 
             // Update operation count
             global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric, 1);
 
             // Update latency for realtime metrics (spawn async task)
             if (metric) < Metric::LastRealtime as usize {
@@ -373,6 +470,7 @@ impl Metrics {
 
                 // Update operation count
                 global_metrics().operations[metric].fetch_add(count as u64, Ordering::Relaxed);
+                emit_otel_counter(metric, count as u64);
 
                 // Update latency for realtime metrics (spawn async task)
                 if (metric) < Metric::LastRealtime as usize {
@@ -408,6 +506,7 @@ impl Metrics {
         let metric = metric as usize;
         // Update operation count
         global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+        emit_otel_counter(metric, 1);
 
         // Update latency for realtime metrics
         if (metric) < Metric::LastRealtime as usize {
@@ -494,10 +593,43 @@ impl Metrics {
         for i in 0..Metric::LastRealtime as usize {
             let last_min = self.latency[i].total().await;
             if last_min.n > 0
-                && let Some(_metric) = Metric::from_index(i)
+                && let Some(metric) = Metric::from_index(i)
             {
-                // Convert to madmin TimedAction format if needed
-                // This would require implementing the conversion
+                metrics.last_minute.actions.insert(
+                    metric.as_str().to_string(),
+                    TimedAction {
+                        count: last_min.n,
+                        acc_time: last_min.total,
+                        bytes: last_min.size,
+                    },
+                );
+            }
+        }
+
+        // Lifetime ILM operations
+        for i in 0..IlmAction::ActionCount as usize {
+            let count = self.actions[i].load(Ordering::Relaxed);
+            if count > 0
+                && let Some(action) = IlmAction::from_index(i)
+            {
+                metrics.life_time_ilm.insert(action.as_str().to_string(), count);
+            }
+        }
+
+        // Last minute ILM latency
+        for i in 0..IlmAction::ActionCount as usize {
+            let last_min = self.actions_latency[i].total().await;
+            if last_min.n > 0
+                && let Some(action) = IlmAction::from_index(i)
+            {
+                metrics.last_minute.ilm.insert(
+                    action.as_str().to_string(),
+                    TimedAction {
+                        count: last_min.n,
+                        acc_time: last_min.total,
+                        bytes: last_min.size,
+                    },
+                );
             }
         }
 
@@ -550,3 +682,31 @@ impl Default for Metrics {
         Self::new()
     }
 }
+
+pub struct CloseDiskGuard(CloseDiskFn);
+
+impl CloseDiskGuard {
+    pub fn new(close_disk: CloseDiskFn) -> Self {
+        Self(close_disk)
+    }
+
+    pub async fn close(&self) {
+        self.0().await;
+    }
+}
+
+impl Drop for CloseDiskGuard {
+    fn drop(&mut self) {
+        // Drop cannot be async, so we spawn the async cleanup task
+        // The task will run in the background and complete asynchronously
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            let close_fn = self.0.clone();
+            handle.spawn(async move {
+                close_fn().await;
+            });
+        } else {
+            // If we're not in a tokio runtime context, we can't spawn
+            // This is a best-effort cleanup, so we just skip it
+        }
+    }
+}

crates/config/AGENTS.md

@@ -0,0 +1,28 @@
+# Config Crate Instructions
+
+Applies to `crates/config/`.
+
+## Environment Variable Naming
+
+- Global configuration variables must use flat `RUSTFS_*` names.
+- Do not introduce module-segmented names such as `RUSTFS_CONFIG_*`.
+
+Canonical examples:
+
+- `RUSTFS_REGION`
+- `RUSTFS_ADDRESS`
+- `RUSTFS_VOLUMES`
+- `RUSTFS_LICENSE`
+- `RUSTFS_SCANNER_ENABLED`
+- `RUSTFS_HEAL_ENABLED`
+
+## Compatibility
+
+- Deprecated aliases must keep warning behavior.
+- Document aliases in `crates/config/README.md`.
+- Any alias change should include tests for both canonical and deprecated forms.
+
+## Source of Truth
+
+- Constants: `crates/config/src/constants/app.rs`
+- Naming conventions: `crates/config/README.md#environment-variable-naming-conventions`

crates/config/README.md

@@ -32,6 +32,30 @@
 
 For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
+## Environment Variable Naming Conventions
+
+RustFS uses a flat naming style for top-level configuration: environment variables are `RUSTFS_*` without nested module segments.
+
+Examples:
+- `RUSTFS_REGION`
+- `RUSTFS_ADDRESS`
+- `RUSTFS_VOLUMES`
+- `RUSTFS_LICENSE`
+
+Current guidance:
+- Prefer module-specific names only when they are not top-level product configuration.
+- Renamed variables must keep backward-compatible aliases until before beta.
+- Alias usage must emit deprecation warnings and be treated as transitional only.
+- Deprecated example:
+  - `RUSTFS_ENABLE_SCANNER` -> `RUSTFS_SCANNER_ENABLED`
+  - `RUSTFS_ENABLE_HEAL` -> `RUSTFS_HEAL_ENABLED`
+  - `RUSTFS_DATA_SCANNER_START_DELAY_SECS` -> `RUSTFS_SCANNER_START_DELAY_SECS`
+
+## Scanner environment aliases
+
+- `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
+- `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
+
 ## 📄 License
 
 This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/config/src/audit/webhook.rs

@@ -20,9 +20,11 @@ pub const ENV_AUDIT_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_WEBHOOK_QUEUE_LIMI
 pub const ENV_AUDIT_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR";
 pub const ENV_AUDIT_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_CERT";
 pub const ENV_AUDIT_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_KEY";
+pub const ENV_AUDIT_WEBHOOK_CLIENT_CA: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_CA";
+pub const ENV_AUDIT_WEBHOOK_SKIP_TLS_VERIFY: &str = "RUSTFS_AUDIT_WEBHOOK_SKIP_TLS_VERIFY";
 
 /// List of all environment variable keys for a webhook target.
-pub const ENV_AUDIT_WEBHOOK_KEYS: &[&str; 7] = &[
+pub const ENV_AUDIT_WEBHOOK_KEYS: &[&str; 9] = &[
     ENV_AUDIT_WEBHOOK_ENABLE,
     ENV_AUDIT_WEBHOOK_ENDPOINT,
     ENV_AUDIT_WEBHOOK_AUTH_TOKEN,
@@ -30,6 +32,8 @@ pub const ENV_AUDIT_WEBHOOK_KEYS: &[&str; 7] = &[
     ENV_AUDIT_WEBHOOK_QUEUE_DIR,
     ENV_AUDIT_WEBHOOK_CLIENT_CERT,
     ENV_AUDIT_WEBHOOK_CLIENT_KEY,
+    ENV_AUDIT_WEBHOOK_CLIENT_CA,
+    ENV_AUDIT_WEBHOOK_SKIP_TLS_VERIFY,
 ];
 
 /// A list of all valid configuration keys for a webhook target.
@@ -42,4 +46,6 @@ pub const AUDIT_WEBHOOK_KEYS: &[&str] = &[
     crate::WEBHOOK_CLIENT_CERT,
     crate::WEBHOOK_CLIENT_KEY,
     crate::COMMENT_KEY,
+    crate::WEBHOOK_CLIENT_CA,
+    crate::WEBHOOK_SKIP_TLS_VERIFY,
 ];

crates/config/src/constants/app.rs

@@ -98,12 +98,45 @@ pub const RUSTFS_HTTP_PREFIX: &str = "http://";
 /// Default value: https://
 pub const RUSTFS_HTTPS_PREFIX: &str = "https://";
 
+/// Default documentation URL for rustfs
+/// This is the default documentation URL for rustfs.
+/// It is used to provide the documentation of the application.
+/// Default value: https://docs.rustfs.com
+pub const RUSTFS_DOCS_URL: &str = "https://docs.rustfs.com";
+
+/// Default GitHub URL for rustfs
+/// This is the default GitHub URL for rustfs.
+/// It is used to provide the source code of the application.
+/// Default value: https://github.com/rustfs/rustfs
+pub const RUSTFS_GITHUB_URL: &str = "https://github.com/rustfs/rustfs";
+
+/// Default license for rustfs
+/// This is the default license for rustfs.
+/// It is used to provide the license of the application.
+/// Default value: Apache-2.0
+pub const RUSTFS_LICENSE: &str = "Apache-2.0";
+
+/// Default license URL for rustfs
+/// This is the default license URL for rustfs.
+/// It is used to provide the license URL of the application.
+/// Default value: https://www.apache.org/licenses/LICENSE-2.0
+pub const RUSTFS_LICENSE_URL: &str = "https://www.apache.org/licenses/LICENSE-2.0";
+
 /// Environment variable for rustfs address
 /// This is the environment variable for rustfs address.
 /// It is used to bind the server to a specific address.
 /// Example: RUSTFS_ADDRESS=":9000"
 pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 
+/// Environment variable for server volumes.
+pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
+
+/// Environment variable for server tls path.
+pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH";
+
+/// Default value for the server TLS path if `ENV_RUSTFS_TLS_PATH` is not set.
+pub const DEFAULT_RUSTFS_TLS_PATH: &str = "";
+
 /// Default port for rustfs
 /// This is the default port for rustfs.
 /// This is used to bind the server to a specific port.
@@ -119,20 +152,34 @@ pub const DEFAULT_CONSOLE_PORT: u16 = 9001;
 
 /// Default address for rustfs console
 /// This is the default address for rustfs console.
+/// This is used to bind the console server to a specific address.
+/// Default value: :9001
 pub const DEFAULT_CONSOLE_ADDRESS: &str = concat!(":", DEFAULT_CONSOLE_PORT);
 
+/// Default region for rustfs
+/// This is the default region for rustfs.
+/// It is used to identify the region of the application.
+/// Default value: us-east-1
+pub const RUSTFS_REGION: &str = "us-east-1";
+
+/// Environment variable for server region.
+pub const ENV_RUSTFS_REGION: &str = "RUSTFS_REGION";
+
+/// Environment variable for server license.
+pub const ENV_RUSTFS_LICENSE: &str = "RUSTFS_LICENSE";
+
 /// Default log filename for rustfs
 /// This is the default log filename for rustfs.
 /// It is used to store the logs of the application.
 /// Default value: rustfs.log
-/// Environment variable: RUSTFS_OBSERVABILITY_LOG_FILENAME
+/// Environment variable: RUSTFS_OBS_LOG_FILENAME
 pub const DEFAULT_LOG_FILENAME: &str = "rustfs";
 
 /// Default OBS log filename for rustfs
 /// This is the default log filename for OBS.
 /// It is used to store the logs of the application.
 /// Default value: rustfs.log
-pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, "");
+pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, ".log");
 
 /// Default log directory for rustfs
 /// This is the default log directory for rustfs.
@@ -151,9 +198,9 @@ pub const DEFAULT_LOG_ROTATION_SIZE_MB: u64 = 100;
 /// Default log rotation time for rustfs
 /// This is the default log rotation time for rustfs.
 /// It is used to rotate the logs of the application.
-/// Default value: hour, eg: day,hour,minute,second
+/// Default value: hour, eg: daily,hourly,minutely
 /// Environment variable: RUSTFS_OBS_LOG_ROTATION_TIME
-pub const DEFAULT_LOG_ROTATION_TIME: &str = "hour";
+pub const DEFAULT_LOG_ROTATION_TIME: &str = "hourly";
 
 /// Default log keep files for rustfs
 /// This is the default log keep files for rustfs.
@@ -162,11 +209,35 @@ pub const DEFAULT_LOG_ROTATION_TIME: &str = "hour";
 /// Environment variable: RUSTFS_OBS_LOG_KEEP_FILES
 pub const DEFAULT_LOG_KEEP_FILES: usize = 30;
 
+/// Default trace export enabled
+/// It is used to enable or disable exporting traces
+/// Default value: true
+/// Environment variable: RUSTFS_OBS_TRACES_EXPORT_ENABLED
+pub const DEFAULT_OBS_TRACES_EXPORT_ENABLED: bool = true;
+
+/// Default metrics export enabled
+/// It is used to enable or disable exporting metrics
+/// Default value: true
+/// Environment variable: RUSTFS_OBS_METRICS_EXPORT_ENABLED
+pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true;
+
+/// Default logs export enabled
+/// It is used to enable or disable exporting logs
+/// Default value: true
+/// Environment variable: RUSTFS_OBS_LOGS_EXPORT_ENABLED
+pub const DEFAULT_OBS_LOGS_EXPORT_ENABLED: bool = true;
+
+/// Default profiling export enabled
+/// It is used to enable or disable exporting profiles
+/// Default value: true
+/// Environment variable: RUSTFS_OBS_PROFILING_EXPORT_ENABLED
+pub const DEFAULT_OBS_PROFILING_EXPORT_ENABLED: bool = true;
+
 /// Default log local logging enabled for rustfs
 /// This is the default log local logging enabled for rustfs.
 /// It is used to enable or disable local logging of the application.
 /// Default value: false
-/// Environment variable: RUSTFS_OBS_LOGL_STDOUT_ENABLED
+/// Environment variable: RUSTFS_OBS_LOG_STDOUT_ENABLED
 pub const DEFAULT_OBS_LOG_STDOUT_ENABLED: bool = false;
 
 /// Constant representing 1 Kibibyte (1024 bytes)

crates/config/src/constants/env.rs

@@ -20,6 +20,8 @@ pub const EVENT_DEFAULT_DIR: &str = "/opt/rustfs/events"; // Default directory f
 pub const AUDIT_DEFAULT_DIR: &str = "/opt/rustfs/audit"; // Default directory for audit store
 pub const DEFAULT_LIMIT: u64 = 100000; // Default store limit
 
+pub const RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT: bool = false;
+
 /// Standard config keys and values.
 pub const ENABLE_KEY: &str = "enable";
 pub const COMMENT_KEY: &str = "comment";

crates/config/src/constants/mod.rs

@@ -19,6 +19,7 @@ pub(crate) mod console;
 pub(crate) mod env;
 pub(crate) mod heal;
 pub(crate) mod object;
+pub(crate) mod oidc;
 pub(crate) mod profiler;
 pub(crate) mod protocols;
 pub(crate) mod proxy;

crates/config/src/constants/object.rs

@@ -167,3 +167,16 @@ pub const DEFAULT_OBJECT_CACHE_TTI_SECS: u64 = 120;
 ///
 /// Default is set to 5 hits.
 pub const DEFAULT_OBJECT_HOT_MIN_HITS_TO_EXTEND: usize = 5;
+
+/// Skip bitrot hash verification on GetObject reads.
+///
+/// When enabled, GetObject reads skip the per-shard hash
+/// computation and comparison, reducing CPU usage on the read path.
+/// The background scanner still performs full integrity verification.
+/// Does not affect writes, heals, or scanner operations.
+///
+/// Default is false (verify on every read, matching pre-existing behavior).
+pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITROT_VERIFY";
+
+/// Default: bitrot verification is enabled on GetObject reads (do not skip).
+pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;

crates/config/src/constants/oidc.rs

@@ -0,0 +1,91 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// OIDC configuration field keys (used in KVS)
+pub const OIDC_CONFIG_URL: &str = "config_url";
+pub const OIDC_CLIENT_ID: &str = "client_id";
+pub const OIDC_CLIENT_SECRET: &str = "client_secret";
+pub const OIDC_SCOPES: &str = "scopes";
+pub const OIDC_REDIRECT_URI: &str = "redirect_uri";
+pub const OIDC_REDIRECT_URI_DYNAMIC: &str = "redirect_uri_dynamic";
+pub const OIDC_CLAIM_NAME: &str = "claim_name";
+pub const OIDC_CLAIM_PREFIX: &str = "claim_prefix";
+pub const OIDC_ROLE_POLICY: &str = "role_policy";
+pub const OIDC_DISPLAY_NAME: &str = "display_name";
+pub const OIDC_GROUPS_CLAIM: &str = "groups_claim";
+pub const OIDC_EMAIL_CLAIM: &str = "email_claim";
+pub const OIDC_USERNAME_CLAIM: &str = "username_claim";
+
+// Environment variable names for OIDC
+pub const ENV_IDENTITY_OPENID_ENABLE: &str = "RUSTFS_IDENTITY_OPENID_ENABLE";
+pub const ENV_IDENTITY_OPENID_CONFIG_URL: &str = "RUSTFS_IDENTITY_OPENID_CONFIG_URL";
+pub const ENV_IDENTITY_OPENID_CLIENT_ID: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_ID";
+pub const ENV_IDENTITY_OPENID_CLIENT_SECRET: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_SECRET";
+pub const ENV_IDENTITY_OPENID_SCOPES: &str = "RUSTFS_IDENTITY_OPENID_SCOPES";
+pub const ENV_IDENTITY_OPENID_REDIRECT_URI: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI";
+pub const ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC";
+pub const ENV_IDENTITY_OPENID_CLAIM_NAME: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_NAME";
+pub const ENV_IDENTITY_OPENID_CLAIM_PREFIX: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_PREFIX";
+pub const ENV_IDENTITY_OPENID_ROLE_POLICY: &str = "RUSTFS_IDENTITY_OPENID_ROLE_POLICY";
+pub const ENV_IDENTITY_OPENID_DISPLAY_NAME: &str = "RUSTFS_IDENTITY_OPENID_DISPLAY_NAME";
+pub const ENV_IDENTITY_OPENID_GROUPS_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM";
+pub const ENV_IDENTITY_OPENID_EMAIL_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM";
+pub const ENV_IDENTITY_OPENID_USERNAME_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM";
+
+/// List of all environment variable keys for an OIDC provider.
+pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
+    ENV_IDENTITY_OPENID_ENABLE,
+    ENV_IDENTITY_OPENID_CONFIG_URL,
+    ENV_IDENTITY_OPENID_CLIENT_ID,
+    ENV_IDENTITY_OPENID_CLIENT_SECRET,
+    ENV_IDENTITY_OPENID_SCOPES,
+    ENV_IDENTITY_OPENID_REDIRECT_URI,
+    ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC,
+    ENV_IDENTITY_OPENID_CLAIM_NAME,
+    ENV_IDENTITY_OPENID_CLAIM_PREFIX,
+    ENV_IDENTITY_OPENID_ROLE_POLICY,
+    ENV_IDENTITY_OPENID_DISPLAY_NAME,
+    ENV_IDENTITY_OPENID_GROUPS_CLAIM,
+    ENV_IDENTITY_OPENID_EMAIL_CLAIM,
+    ENV_IDENTITY_OPENID_USERNAME_CLAIM,
+];
+
+/// A list of all valid configuration keys for an OIDC provider.
+pub const IDENTITY_OPENID_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    OIDC_CONFIG_URL,
+    OIDC_CLIENT_ID,
+    OIDC_CLIENT_SECRET,
+    OIDC_SCOPES,
+    OIDC_REDIRECT_URI,
+    OIDC_REDIRECT_URI_DYNAMIC,
+    OIDC_CLAIM_NAME,
+    OIDC_CLAIM_PREFIX,
+    OIDC_ROLE_POLICY,
+    OIDC_DISPLAY_NAME,
+    OIDC_GROUPS_CLAIM,
+    OIDC_EMAIL_CLAIM,
+    OIDC_USERNAME_CLAIM,
+    crate::COMMENT_KEY,
+];
+
+// Default values
+pub const OIDC_DEFAULT_SCOPES: &str = "openid,profile,email";
+pub const OIDC_DEFAULT_CLAIM_NAME: &str = "groups";
+pub const OIDC_DEFAULT_GROUPS_CLAIM: &str = "groups";
+pub const OIDC_DEFAULT_EMAIL_CLAIM: &str = "email";
+pub const OIDC_DEFAULT_USERNAME_CLAIM: &str = "preferred_username";
+
+// Subsystem identifier
+pub const IDENTITY_OPENID_SUB_SYS: &str = "identity_openid";

crates/config/src/constants/protocols.rs

@@ -45,3 +45,15 @@ pub const ENV_FTPS_CERTS_DIR: &str = "RUSTFS_FTPS_CERTS_DIR";
 pub const ENV_FTPS_CA_FILE: &str = "RUSTFS_FTPS_CA_FILE";
 pub const ENV_FTPS_PASSIVE_PORTS: &str = "RUSTFS_FTPS_PASSIVE_PORTS";
 pub const ENV_FTPS_EXTERNAL_IP: &str = "RUSTFS_FTPS_EXTERNAL_IP";
+
+/// Default WebDAV server bind address
+pub const DEFAULT_WEBDAV_ADDRESS: &str = "0.0.0.0:8080";
+
+/// WebDAV environment variable names
+pub const ENV_WEBDAV_ENABLE: &str = "RUSTFS_WEBDAV_ENABLE";
+pub const ENV_WEBDAV_ADDRESS: &str = "RUSTFS_WEBDAV_ADDRESS";
+pub const ENV_WEBDAV_TLS_ENABLED: &str = "RUSTFS_WEBDAV_TLS_ENABLED";
+pub const ENV_WEBDAV_CERTS_DIR: &str = "RUSTFS_WEBDAV_CERTS_DIR";
+pub const ENV_WEBDAV_CA_FILE: &str = "RUSTFS_WEBDAV_CA_FILE";
+pub const ENV_WEBDAV_MAX_BODY_SIZE: &str = "RUSTFS_WEBDAV_MAX_BODY_SIZE";
+pub const ENV_WEBDAV_REQUEST_TIMEOUT: &str = "RUSTFS_WEBDAV_REQUEST_TIMEOUT";

crates/config/src/constants/scanner.rs

@@ -12,17 +12,107 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/// Environment variable name that specifies the data scanner start delay in seconds.
-/// - Purpose: Define the delay between data scanner operations.
+use std::time::Duration;
+
+/// Canonical environment variable name that specifies the scanner start delay in seconds.
+/// If set, this overrides the cycle interval derived from `RUSTFS_SCANNER_SPEED`.
 /// - Unit: seconds (u64).
-/// - Valid values: any positive integer.
-/// - Semantics: This delay controls how frequently the data scanner checks for and processes data; shorter delays lead to more responsive scanning but may increase system load.
-/// - Example: `export RUSTFS_DATA_SCANNER_START_DELAY_SECS=10`
-/// - Note: Choose an appropriate delay that balances scanning responsiveness with overall system performance.
+/// - Example: `export RUSTFS_SCANNER_START_DELAY_SECS=10`
+pub const ENV_SCANNER_START_DELAY_SECS: &str = "RUSTFS_SCANNER_START_DELAY_SECS";
+
+/// Deprecated compatibility alias for scanner start delay.
+/// Prefer `RUSTFS_SCANNER_START_DELAY_SECS`.
+#[deprecated(note = "Use RUSTFS_SCANNER_START_DELAY_SECS instead")]
 pub const ENV_DATA_SCANNER_START_DELAY_SECS: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
 
-/// Default data scanner start delay in seconds if not specified in the environment variable.
-/// - Value: 10 seconds.
-/// - Rationale: This default interval provides a reasonable balance between scanning responsiveness and system load for most deployments.
-/// - Adjustments: Users may modify this value via the `RUSTFS_DATA_SCANNER_START_DELAY_SECS` environment variable based on their specific scanning requirements and system performance.
-pub const DEFAULT_DATA_SCANNER_START_DELAY_SECS: u64 = 60;
+/// Environment variable that selects the scanner speed preset.
+/// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
+/// Controls the sleep factor, maximum sleep duration, and cycle interval.
+/// - Example: `export RUSTFS_SCANNER_SPEED=slow`
+pub const ENV_SCANNER_SPEED: &str = "RUSTFS_SCANNER_SPEED";
+
+/// Default scanner speed preset.
+pub const DEFAULT_SCANNER_SPEED: &str = "default";
+
+/// Environment variable that controls whether the scanner sleeps between operations.
+/// When `true` (default), the scanner throttles itself. When `false`, it runs at full speed.
+/// - Example: `export RUSTFS_SCANNER_IDLE_MODE=false`
+pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE";
+
+/// Default scanner idle mode.
+pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
+
+/// Scanner speed preset controlling throttling behavior.
+///
+/// Each preset defines three parameters:
+/// - **sleep_factor**: Multiplier applied to elapsed work time to compute inter-object sleep.
+/// - **max_sleep**: Upper bound on any single throttle sleep.
+/// - **cycle_interval**: Base delay between scan cycles.
+///
+/// | Preset    | Factor | Max Sleep | Cycle Interval |
+/// |-----------|--------|-----------|----------------|
+/// | `fastest` | 0      | 0         | 1 second       |
+/// | `fast`    | 1x     | 100ms     | 1 minute       |
+/// | `default` | 2x     | 1 second  | 1 minute       |
+/// | `slow`    | 10x    | 15 seconds| 1 minute       |
+/// | `slowest` | 100x   | 15 seconds| 30 minutes     |
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum ScannerSpeed {
+    Fastest,
+    Fast,
+    #[default]
+    Default,
+    Slow,
+    Slowest,
+}
+
+impl ScannerSpeed {
+    pub fn sleep_factor(self) -> f64 {
+        match self {
+            Self::Fastest => 0.0,
+            Self::Fast => 1.0,
+            Self::Default => 2.0,
+            Self::Slow => 10.0,
+            Self::Slowest => 100.0,
+        }
+    }
+
+    pub fn max_sleep(self) -> Duration {
+        match self {
+            Self::Fastest => Duration::ZERO,
+            Self::Fast => Duration::from_millis(100),
+            Self::Default => Duration::from_secs(1),
+            Self::Slow | Self::Slowest => Duration::from_secs(15),
+        }
+    }
+
+    pub fn cycle_interval(self) -> Duration {
+        match self {
+            Self::Fastest => Duration::from_secs(1),
+            Self::Fast | Self::Default | Self::Slow => Duration::from_secs(60),
+            Self::Slowest => Duration::from_secs(30 * 60),
+        }
+    }
+
+    pub fn from_env_str(s: &str) -> Self {
+        match s.trim().to_ascii_lowercase().as_str() {
+            "fastest" => Self::Fastest,
+            "fast" => Self::Fast,
+            "slow" => Self::Slow,
+            "slowest" => Self::Slowest,
+            _ => Self::Default,
+        }
+    }
+}
+
+impl std::fmt::Display for ScannerSpeed {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Fastest => write!(f, "fastest"),
+            Self::Fast => write!(f, "fast"),
+            Self::Default => write!(f, "default"),
+            Self::Slow => write!(f, "slow"),
+            Self::Slowest => write!(f, "slowest"),
+        }
+    }
+}

crates/config/src/constants/targets.rs

@@ -16,6 +16,8 @@ pub const WEBHOOK_ENDPOINT: &str = "endpoint";
 pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token";
 pub const WEBHOOK_CLIENT_CERT: &str = "client_cert";
 pub const WEBHOOK_CLIENT_KEY: &str = "client_key";
+pub const WEBHOOK_CLIENT_CA: &str = "client_ca";
+pub const WEBHOOK_SKIP_TLS_VERIFY: &str = "skip_tls_verify";
 pub const WEBHOOK_BATCH_SIZE: &str = "batch_size";
 pub const WEBHOOK_QUEUE_LIMIT: &str = "queue_limit";
 pub const WEBHOOK_QUEUE_DIR: &str = "queue_dir";

crates/config/src/lib.rs

@@ -44,6 +44,10 @@ pub use constants::scanner::*;
 pub use constants::targets::*;
 #[cfg(feature = "constants")]
 pub use constants::tls::*;
+#[cfg(feature = "constants")]
+pub mod oidc {
+    pub use super::constants::oidc::*;
+}
 #[cfg(feature = "audit")]
 pub mod audit;
 #[cfg(feature = "notify")]

crates/config/src/notify/webhook.rs

@@ -22,6 +22,8 @@ pub const NOTIFY_WEBHOOK_KEYS: &[&str] = &[
     crate::WEBHOOK_CLIENT_CERT,
     crate::WEBHOOK_CLIENT_KEY,
     crate::COMMENT_KEY,
+    crate::WEBHOOK_CLIENT_CA,
+    crate::WEBHOOK_SKIP_TLS_VERIFY,
 ];
 
 // Webhook Environment Variables
@@ -32,8 +34,10 @@ pub const ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_LI
 pub const ENV_NOTIFY_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR";
 pub const ENV_NOTIFY_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CERT";
 pub const ENV_NOTIFY_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_KEY";
+pub const ENV_NOTIFY_WEBHOOK_CLIENT_CA: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CA";
+pub const ENV_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY: &str = "RUSTFS_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY";
 
-pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 7] = &[
+pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 9] = &[
     ENV_NOTIFY_WEBHOOK_ENABLE,
     ENV_NOTIFY_WEBHOOK_ENDPOINT,
     ENV_NOTIFY_WEBHOOK_AUTH_TOKEN,
@@ -41,4 +45,6 @@ pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 7] = &[
     ENV_NOTIFY_WEBHOOK_QUEUE_DIR,
     ENV_NOTIFY_WEBHOOK_CLIENT_CERT,
     ENV_NOTIFY_WEBHOOK_CLIENT_KEY,
+    ENV_NOTIFY_WEBHOOK_CLIENT_CA,
+    ENV_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY,
 ];

crates/config/src/observability/mod.rs

@@ -15,43 +15,60 @@
 // Observability Keys
 
 mod metrics;
+use const_str::concat;
 pub use metrics::*;
 
 pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
 pub const ENV_OBS_TRACE_ENDPOINT: &str = "RUSTFS_OBS_TRACE_ENDPOINT";
 pub const ENV_OBS_METRIC_ENDPOINT: &str = "RUSTFS_OBS_METRIC_ENDPOINT";
 pub const ENV_OBS_LOG_ENDPOINT: &str = "RUSTFS_OBS_LOG_ENDPOINT";
+pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT";
 pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
 pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
 pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
 pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
 pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
 pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
+
+/// Per-signal enable/disable flags (default: true)
+pub const ENV_OBS_TRACES_EXPORT_ENABLED: &str = "RUSTFS_OBS_TRACES_EXPORT_ENABLED";
+pub const ENV_OBS_METRICS_EXPORT_ENABLED: &str = "RUSTFS_OBS_METRICS_EXPORT_ENABLED";
+pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED";
+pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ENABLED";
+
 pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
 pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
 pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
 pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
-pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB";
 pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
 pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
 
-/// Log pool capacity for async logging
-pub const ENV_OBS_LOG_POOL_CAPA: &str = "RUSTFS_OBS_LOG_POOL_CAPA";
+/// Log cleanup related configurations
+pub const ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES";
+pub const ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES";
+pub const ENV_OBS_LOG_COMPRESS_OLD_FILES: &str = "RUSTFS_OBS_LOG_COMPRESS_OLD_FILES";
+pub const ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL: &str = "RUSTFS_OBS_LOG_GZIP_COMPRESSION_LEVEL";
+pub const ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS: &str = "RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS";
+pub const ENV_OBS_LOG_EXCLUDE_PATTERNS: &str = "RUSTFS_OBS_LOG_EXCLUDE_PATTERNS";
+pub const ENV_OBS_LOG_DELETE_EMPTY_FILES: &str = "RUSTFS_OBS_LOG_DELETE_EMPTY_FILES";
+pub const ENV_OBS_LOG_MIN_FILE_AGE_SECONDS: &str = "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS";
+pub const ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS: &str = "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS";
+pub const ENV_OBS_LOG_DRY_RUN: &str = "RUSTFS_OBS_LOG_DRY_RUN";
+pub const ENV_OBS_LOG_MATCH_MODE: &str = "RUSTFS_OBS_LOG_MATCH_MODE";
 
-/// Log message capacity for async logging
-pub const ENV_OBS_LOG_MESSAGE_CAPA: &str = "RUSTFS_OBS_LOG_MESSAGE_CAPA";
-
-/// Log flush interval in milliseconds for async logging
-pub const ENV_OBS_LOG_FLUSH_MS: &str = "RUSTFS_OBS_LOG_FLUSH_MS";
-
-/// Default values for log pool
-pub const DEFAULT_OBS_LOG_POOL_CAPA: usize = 10240;
-
-/// Default values for message capacity
-pub const DEFAULT_OBS_LOG_MESSAGE_CAPA: usize = 32768;
-
-/// Default values for flush interval in milliseconds
-pub const DEFAULT_OBS_LOG_FLUSH_MS: u64 = 200;
+/// Default values for log cleanup
+pub const DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES: u64 = 2 * 1024 * 1024 * 1024; // 2 GiB
+pub const DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: u64 = 0; // No single file limit
+pub const DEFAULT_OBS_LOG_COMPRESS_OLD_FILES: bool = true;
+pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL: u32 = 6;
+pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_EXTENSION: &str = "gz";
+pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION: &str = concat!(".", DEFAULT_OBS_LOG_GZIP_COMPRESSION_EXTENSION);
+pub const DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS: u64 = 30; // Retain compressed files for 30 days
+pub const DEFAULT_OBS_LOG_DELETE_EMPTY_FILES: bool = true;
+pub const DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS: u64 = 3600; // 1 hour
+pub const DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS: u64 = 1800; // 0.5 hours
+pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false;
+pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix";
 
 /// Default values for observability configuration
 // ### Supported Environment Values
@@ -74,6 +91,7 @@ mod tests {
         assert_eq!(ENV_OBS_TRACE_ENDPOINT, "RUSTFS_OBS_TRACE_ENDPOINT");
         assert_eq!(ENV_OBS_METRIC_ENDPOINT, "RUSTFS_OBS_METRIC_ENDPOINT");
         assert_eq!(ENV_OBS_LOG_ENDPOINT, "RUSTFS_OBS_LOG_ENDPOINT");
+        assert_eq!(ENV_OBS_PROFILING_ENDPOINT, "RUSTFS_OBS_PROFILING_ENDPOINT");
         assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
         assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");
         assert_eq!(ENV_OBS_METER_INTERVAL, "RUSTFS_OBS_METER_INTERVAL");
@@ -84,9 +102,27 @@ mod tests {
         assert_eq!(ENV_OBS_LOG_STDOUT_ENABLED, "RUSTFS_OBS_LOG_STDOUT_ENABLED");
         assert_eq!(ENV_OBS_LOG_DIRECTORY, "RUSTFS_OBS_LOG_DIRECTORY");
         assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
-        assert_eq!(ENV_OBS_LOG_ROTATION_SIZE_MB, "RUSTFS_OBS_LOG_ROTATION_SIZE_MB");
         assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
         assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
+        assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED");
+        assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
+        assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
+        assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
+        // Test log cleanup related env keys
+        assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
+        assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
+        assert_eq!(ENV_OBS_LOG_COMPRESS_OLD_FILES, "RUSTFS_OBS_LOG_COMPRESS_OLD_FILES");
+        assert_eq!(ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, "RUSTFS_OBS_LOG_GZIP_COMPRESSION_LEVEL");
+        assert_eq!(
+            ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
+            "RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS"
+        );
+        assert_eq!(ENV_OBS_LOG_EXCLUDE_PATTERNS, "RUSTFS_OBS_LOG_EXCLUDE_PATTERNS");
+        assert_eq!(ENV_OBS_LOG_DELETE_EMPTY_FILES, "RUSTFS_OBS_LOG_DELETE_EMPTY_FILES");
+        assert_eq!(ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS");
+        assert_eq!(ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS");
+        assert_eq!(ENV_OBS_LOG_DRY_RUN, "RUSTFS_OBS_LOG_DRY_RUN");
+        assert_eq!(ENV_OBS_LOG_MATCH_MODE, "RUSTFS_OBS_LOG_MATCH_MODE");
     }
 
     #[test]
@@ -95,5 +131,6 @@ mod tests {
         assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
         assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
         assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
+        assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix");
     }
 }

crates/credentials/src/credentials.rs

@@ -233,15 +233,18 @@ impl<'a> fmt::Debug for Masked<'a> {
         match self.0 {
             None => Ok(()),
             Some(s) => {
-                let len = s.len();
+                let len = s.chars().count();
                 if len == 0 {
                     Ok(())
                 } else if len == 1 {
                     write!(f, "***")
                 } else if len == 2 {
-                    write!(f, "{}***|{}", &s[0..1], len)
+                    let first = s.chars().next().ok_or(fmt::Error)?;
+                    write!(f, "{}***|{}", first, len)
                 } else {
-                    write!(f, "{}***{}|{}", &s[0..1], &s[len - 1..], len)
+                    let first = s.chars().next().ok_or(fmt::Error)?;
+                    let last = s.chars().last().ok_or(fmt::Error)?;
+                    write!(f, "{}***{}|{}", first, last, len)
                 }
             }
         }
@@ -482,5 +485,10 @@ mod tests {
 
         // Test longer string
         assert_eq!(format!("{:?}", Masked(Some("secretpassword"))), "s***d|14");
+
+        // Test Unicode input should not panic and should keep character boundary
+        assert_eq!(format!("{:?}", Masked(Some("中"))), "***");
+        assert_eq!(format!("{:?}", Masked(Some("中文"))), "中***|2");
+        assert_eq!(format!("{:?}", Masked(Some("中文测试"))), "中***试|4");
     }
 }

crates/e2e_test/AGENTS.md

@@ -0,0 +1,26 @@
+# E2E Test Crate Instructions
+
+Applies to `crates/e2e_test/`.
+
+## Test Reliability
+
+- Keep end-to-end tests deterministic and environment-aware.
+- Prefer readiness checks and explicit polling over fixed sleep-based timing assumptions.
+- Ensure tests isolate resources and clean up temporary state.
+
+## Environment Safety
+
+- For local KMS-related E2E runs, keep proxy bypass settings:
+  - `NO_PROXY=127.0.0.1,localhost`
+  - clear `HTTP_PROXY` and `HTTPS_PROXY`
+- Do not hardcode machine-specific paths or credentials.
+
+## Scope and Cost
+
+- Place exhaustive integration behavior here; keep unit behavior in source crates.
+- Keep new E2E scenarios focused and avoid redundant overlap with existing suites.
+
+## Suggested Validation
+
+- `cargo test --package e2e_test`
+- Full gate before commit: `make pre-commit`

crates/e2e_test/Cargo.toml

@@ -59,5 +59,4 @@ sha2 = { workspace = true }
 suppaftp = { workspace = true, features = ["tokio", "rustls-aws-lc-rs"] }
 rcgen.workspace = true
 anyhow.workspace = true
-rustls.workspace = true
-rustls-pemfile.workspace = true
+rustls.workspace = true

crates/e2e_test/src/anonymous_access_test.rs

@@ -0,0 +1,172 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression tests for Issue #2036
+//! Verifies that anonymous access works correctly with bucket policies
+//! when PublicAccessBlock configuration is missing or explicitly set.
+
+use crate::common::{RustFSTestEnvironment, init_logging};
+use aws_sdk_s3::types::PublicAccessBlockConfiguration;
+use serial_test::serial;
+use tracing::info;
+
+async fn setup_public_bucket(
+    env: &RustFSTestEnvironment,
+    bucket_name: &str,
+) -> Result<aws_sdk_s3::Client, Box<dyn std::error::Error + Send + Sync>> {
+    let admin_client = env.create_s3_client();
+
+    admin_client.create_bucket().bucket(bucket_name).send().await?;
+
+    let policy_json = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Sid": "AllowAnonymousGetObject",
+                "Effect": "Allow",
+                "Principal": "*",
+                "Action": ["s3:GetObject"],
+                "Resource": [format!("arn:aws:s3:::{}/*", bucket_name)]
+            }
+        ]
+    })
+    .to_string();
+
+    admin_client
+        .put_bucket_policy()
+        .bucket(bucket_name)
+        .policy(&policy_json)
+        .send()
+        .await?;
+
+    admin_client
+        .put_object()
+        .bucket(bucket_name)
+        .key("test.txt")
+        .body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello anonymous"))
+        .send()
+        .await?;
+
+    Ok(admin_client)
+}
+
+async fn anonymous_get_object(
+    env: &RustFSTestEnvironment,
+    bucket_name: &str,
+    key: &str,
+) -> Result<reqwest::Response, reqwest::Error> {
+    let url = format!("{}/{}/{}", env.url, bucket_name, key);
+    reqwest::Client::new().get(&url).send().await
+}
+
+/// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it
+/// and no PublicAccessBlock configuration exists (ConfigNotFound).
+#[tokio::test]
+#[serial]
+async fn test_anonymous_access_allowed_when_public_access_block_missing() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
+{
+    init_logging();
+    info!("Starting test: anonymous access with missing PublicAccessBlock config...");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket_name = "anon-test-no-pab";
+    let admin_client = setup_public_bucket(&env, bucket_name).await?;
+
+    let _ = admin_client.delete_public_access_block().bucket(bucket_name).send().await;
+
+    let resp = anonymous_get_object(&env, bucket_name, "test.txt").await?;
+    assert_eq!(
+        resp.status().as_u16(),
+        200,
+        "Anonymous GetObject should succeed when no PublicAccessBlock config exists"
+    );
+
+    info!("Test passed: anonymous access allowed with missing PublicAccessBlock config");
+    Ok(())
+}
+
+/// Anonymous GetObject should be denied when RestrictPublicBuckets is true.
+#[tokio::test]
+#[serial]
+async fn test_anonymous_access_denied_when_restrict_public_buckets_enabled()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    info!("Starting test: anonymous access denied with RestrictPublicBuckets=true...");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket_name = "anon-test-restrict";
+    let admin_client = setup_public_bucket(&env, bucket_name).await?;
+
+    admin_client
+        .put_public_access_block()
+        .bucket(bucket_name)
+        .public_access_block_configuration(
+            PublicAccessBlockConfiguration::builder()
+                .restrict_public_buckets(true)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let resp = anonymous_get_object(&env, bucket_name, "test.txt").await?;
+    assert_eq!(
+        resp.status().as_u16(),
+        403,
+        "Anonymous GetObject should be denied when RestrictPublicBuckets is true"
+    );
+
+    info!("Test passed: anonymous access denied with RestrictPublicBuckets=true");
+    Ok(())
+}
+
+/// Anonymous GetObject should succeed when PublicAccessBlock exists but
+/// RestrictPublicBuckets is explicitly false.
+#[tokio::test]
+#[serial]
+async fn test_anonymous_access_allowed_when_restrict_public_buckets_disabled()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    info!("Starting test: anonymous access allowed with RestrictPublicBuckets=false...");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket_name = "anon-test-allow";
+    let admin_client = setup_public_bucket(&env, bucket_name).await?;
+
+    admin_client
+        .put_public_access_block()
+        .bucket(bucket_name)
+        .public_access_block_configuration(
+            PublicAccessBlockConfiguration::builder()
+                .restrict_public_buckets(false)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let resp = anonymous_get_object(&env, bucket_name, "test.txt").await?;
+    assert_eq!(
+        resp.status().as_u16(),
+        200,
+        "Anonymous GetObject should succeed when RestrictPublicBuckets is false"
+    );
+
+    info!("Test passed: anonymous access allowed with RestrictPublicBuckets=false");
+    Ok(())
+}

crates/e2e_test/src/content_encoding_test.rs

@@ -82,4 +82,60 @@ mod tests {
 
         env.stop_server();
     }
+
+    /// Issue #1857: Content-Encoding "aws-chunked" is used by SigV4 streaming clients and must
+    /// not be stored or returned. Upload with aws-chunked and verify GET/HEAD do not return it.
+    #[tokio::test]
+    #[serial]
+    async fn test_content_encoding_aws_chunked_not_returned_issue_1857() {
+        init_logging();
+        info!("Issue #1857: aws-chunked must not be persisted or returned");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "content-encoding-aws-chunked-test";
+        let key = "streamed/object.bin";
+        let content = b"streaming upload body";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .content_encoding("aws-chunked")
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("PUT failed");
+
+        let get_resp = client.get_object().bucket(bucket).key(key).send().await.expect("GET failed");
+        assert_eq!(
+            get_resp.content_encoding(),
+            None,
+            "GET must not return Content-Encoding when only aws-chunked was sent (Issue #1857)"
+        );
+
+        let head_resp = client
+            .head_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("HEAD failed");
+        assert_eq!(
+            head_resp.content_encoding(),
+            None,
+            "HEAD must not return Content-Encoding when only aws-chunked was sent (Issue #1857)"
+        );
+
+        env.stop_server();
+    }
 }

crates/e2e_test/src/delete_objects_versioning_test.rs

@@ -0,0 +1,288 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for Issue #1878:
+//! "In a versioned Bucket, DeleteMarkers are not appearing straight after
+//! a delete_objects is called."
+//!
+//! Root cause: `delete_versions_internal` wrote new xl.meta to disk via
+//! `write_all_private` without invalidating the `GlobalFileCache`. Subsequent
+//! calls to `read_metadata` returned the stale cached xl.meta (without the
+//! delete marker), making `list_object_versions` show the old version as
+//! `IsLatest=true` rather than the new delete marker.
+//!
+//! Fix: `write_all_private` now calls `get_global_file_cache().invalidate()`
+//! after every successful write, and `rename_data` also invalidates the cache
+//! for the destination path after the atomic rename.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
+    use serial_test::serial;
+    use tracing::info;
+
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    /// Regression test for Issue #1878.
+    ///
+    /// Verifies that after calling `delete_objects` (the batch plural API) on
+    /// a versioned bucket, calling `list_object_versions` **immediately** (with
+    /// no sleep) returns the newly-created DeleteMarker with `is_latest = true`.
+    #[tokio::test]
+    #[serial]
+    async fn test_delete_objects_delete_marker_immediately_visible() {
+        init_logging();
+        info!("🧪 TEST: DeleteMarker from delete_objects is immediately visible via list_object_versions");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-delete-marker-visibility";
+        let key = "test-prefix/test-object.txt";
+
+        // 1. Create bucket
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+        info!("✅ Bucket created: {}", bucket);
+
+        // 2. Enable versioning
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to enable versioning");
+        info!("✅ Versioning enabled");
+
+        // 3. Put an object
+        let put_resp = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello versioned world\n"))
+            .send()
+            .await
+            .expect("Failed to put object");
+        let original_version_id = put_resp.version_id().map(str::to_string);
+        info!("✅ Object put, version_id: {:?}", original_version_id);
+        assert!(
+            original_version_id.is_some(),
+            "PutObject should return a version_id in a versioned bucket"
+        );
+
+        // 4. List versions – should show exactly 1 Version with is_latest = true
+        let list_before = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions before delete");
+
+        let versions_before = list_before.versions().to_vec();
+        let markers_before = list_before.delete_markers().to_vec();
+        assert_eq!(versions_before.len(), 1, "Should have 1 version before delete");
+        assert!(markers_before.is_empty(), "Should have no delete markers before delete");
+        assert_eq!(
+            versions_before[0].is_latest(),
+            Some(true),
+            "Original version should be latest before delete"
+        );
+        info!("✅ Verified: 1 version, 0 delete markers before delete_objects");
+
+        // 5. Call delete_objects (plural, no version_id → creates a delete marker)
+        let del_resp = client
+            .delete_objects()
+            .bucket(bucket)
+            .delete(
+                Delete::builder()
+                    .objects(
+                        ObjectIdentifier::builder()
+                            .key(key)
+                            .build()
+                            .expect("Failed to build object identifier"),
+                    )
+                    .build()
+                    .expect("Failed to build delete request"),
+            )
+            .send()
+            .await
+            .expect("Failed to call delete_objects");
+
+        let deleted = del_resp.deleted();
+        assert_eq!(deleted.len(), 1, "delete_objects should report 1 deleted object");
+        let delete_marker_version_id = deleted[0].delete_marker_version_id().map(str::to_string);
+        assert!(
+            deleted[0].delete_marker() == Some(true),
+            "delete_objects should report delete_marker=true for versioned delete"
+        );
+        assert!(
+            delete_marker_version_id.is_some(),
+            "delete_objects should return a delete_marker_version_id"
+        );
+        info!("✅ delete_objects succeeded, delete_marker_version_id: {:?}", delete_marker_version_id);
+
+        // 6. List versions IMMEDIATELY (no sleep!) – this is the key regression assertion.
+        //    Before the fix, the stale file cache caused the listing to return the original
+        //    version as is_latest=true without any delete marker.
+        let list_after = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions immediately after delete_objects");
+
+        let versions_after = list_after.versions().to_vec();
+        let markers_after = list_after.delete_markers().to_vec();
+
+        info!("Versions after delete_objects: {:?}", versions_after.len());
+        info!("Delete markers after delete_objects: {:?}", markers_after.len());
+
+        // The delete marker must be visible immediately – no delay required.
+        assert_eq!(
+            markers_after.len(),
+            1,
+            "list_object_versions should return exactly 1 delete marker IMMEDIATELY after delete_objects (regression #1878)"
+        );
+        assert_eq!(markers_after[0].is_latest(), Some(true), "The delete marker should be is_latest=true");
+        assert_eq!(
+            markers_after[0].version_id().map(str::to_string),
+            delete_marker_version_id,
+            "The delete marker version_id should match the one returned by delete_objects"
+        );
+
+        // The original version should still be present, but is_latest = false
+        assert_eq!(versions_after.len(), 1, "The original version should still be present");
+        assert_eq!(
+            versions_after[0].is_latest(),
+            Some(false),
+            "Original version should be is_latest=false after delete marker is created"
+        );
+
+        info!("✅ TEST PASSED: Delete marker is immediately visible after delete_objects (no sleep needed)");
+    }
+
+    /// Additional regression check: verify that multiple objects deleted via
+    /// a single `delete_objects` call all have their delete markers visible
+    /// immediately afterwards.
+    #[tokio::test]
+    #[serial]
+    async fn test_delete_objects_multiple_keys_delete_markers_immediately_visible() {
+        init_logging();
+        info!("🧪 TEST: Multiple delete markers from delete_objects are immediately visible");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-multi-delete-marker-visibility";
+        let keys = ["obj1.txt", "obj2.txt", "obj3.txt"];
+
+        // Create bucket and enable versioning
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to enable versioning");
+
+        // Put all objects
+        for key in &keys {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(*key)
+                .body(aws_sdk_s3::primitives::ByteStream::from_static(b"data"))
+                .send()
+                .await
+                .unwrap_or_else(|_| panic!("Failed to put object {}", key));
+        }
+        info!("✅ Put {} objects", keys.len());
+
+        // Delete all objects with a single delete_objects call
+        let identifiers: Vec<ObjectIdentifier> = keys
+            .iter()
+            .map(|k| {
+                ObjectIdentifier::builder()
+                    .key(*k)
+                    .build()
+                    .expect("Failed to build ObjectIdentifier")
+            })
+            .collect();
+
+        let del_resp = client
+            .delete_objects()
+            .bucket(bucket)
+            .delete(
+                Delete::builder()
+                    .set_objects(Some(identifiers))
+                    .build()
+                    .expect("Failed to build delete"),
+            )
+            .send()
+            .await
+            .expect("Failed to call delete_objects");
+
+        assert_eq!(del_resp.deleted().len(), keys.len(), "All keys should be reported deleted");
+        info!("✅ delete_objects deleted {} objects", keys.len());
+
+        // Immediately list all versions and check each key has a delete marker
+        let list_resp = client
+            .list_object_versions()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to list object versions");
+
+        let markers = list_resp.delete_markers().to_vec();
+        assert_eq!(
+            markers.len(),
+            keys.len(),
+            "Each deleted key should have an immediately-visible delete marker (regression #1878)"
+        );
+
+        for marker in &markers {
+            assert_eq!(marker.is_latest(), Some(true), "Each delete marker should be is_latest=true");
+        }
+
+        info!("✅ TEST PASSED: All {} delete markers are immediately visible", keys.len());
+    }
+}

crates/e2e_test/src/group_delete_test.rs

@@ -0,0 +1,208 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! E2E tests for group management (fixes #2028).
+
+use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_put, init_logging};
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::{Client, Config};
+use serial_test::serial;
+use tracing::info;
+
+fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-group-test");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+/// Test that deleting a group with members fails, and deleting an empty group succeeds.
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+#[ignore = "requires awscurl and spawns a real RustFS server"]
+async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    // 1. Create a user
+    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
+    let user_body = serde_json::json!({
+        "secretKey": "testuser1secret",
+        "status": "enabled"
+    });
+    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created testuser1");
+
+    // 2. Create a group with testuser1 as a member
+    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let add_member_body = serde_json::json!({
+        "group": "testgroup",
+        "members": ["testuser1"],
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Added testuser1 to testgroup");
+
+    // 3. Attempt to delete the group while it still has members — should fail
+    let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
+    let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
+    assert!(delete_result.is_err(), "deleting a non-empty group should fail");
+    info!("Delete of non-empty group correctly rejected");
+
+    // 4. Remove the member from the group
+    let remove_member_body = serde_json::json!({
+        "group": "testgroup",
+        "members": ["testuser1"],
+        "isRemove": true,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Removed testuser1 from testgroup");
+
+    // 5. Delete the now-empty group — should succeed
+    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    info!("Deleted empty testgroup successfully");
+
+    // 6. Verify the group no longer exists
+    let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
+    let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
+    assert!(get_result.is_err(), "group should no longer exist after deletion");
+    info!("Confirmed testgroup no longer exists");
+
+    Ok(())
+}
+
+/// Test that a user with only group membership (no explicit user policy) gets group policies
+/// and can perform actions allowed by the group (regression test for #2028.1).
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+#[ignore = "requires awscurl and spawns a real RustFS server"]
+async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let user_name = "grouponlyuser";
+    let user_secret = "grouponlysecret";
+    let group_name = "policygroup";
+    let policy_name = "ListBucketsOnlyPolicy";
+
+    // 1. Create canned policy that allows ListAllMyBuckets only
+    let policy_doc = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:ListAllMyBuckets"],
+            "Resource": ["*"]
+        }]
+    });
+    let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created canned policy {}", policy_name);
+
+    // 2. Create user with no explicit policy
+    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
+    let user_body = serde_json::json!({
+        "secretKey": user_secret,
+        "status": "enabled"
+    });
+    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created user {} with no explicit policy", user_name);
+
+    // 3. Add user to group (creates group with this member; user_group_memberships must be updated)
+    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let add_member_body = serde_json::json!({
+        "group": group_name,
+        "members": [user_name],
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Added {} to group {}", user_name, group_name);
+
+    // 4. Attach policy to group
+    let set_policy_url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
+        env.url, policy_name, group_name
+    );
+    awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
+    info!("Attached policy {} to group {}", policy_name, group_name);
+
+    // 5. User with only group (no user policy) should be able to list buckets
+    let user_client = create_user_s3_client(&env, user_name, user_secret);
+    let list_result = user_client.list_buckets().send().await;
+    list_result.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> {
+        format!("User with only group membership should get group policies and list buckets: {}", e).into()
+    })?;
+    info!("User with only group successfully listed buckets (group policies applied)");
+
+    Ok(())
+}
+
+/// Test that after deleting a user who was the only member of a group, the group can be deleted
+/// (regression test for #2028.2: delete group uses backend membership, not stale cache).
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+#[ignore = "requires awscurl and spawns a real RustFS server"]
+async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let user_name = "solemember";
+    let user_secret = "solemembersecret";
+    let group_name = "soledeletegroup";
+
+    // 1. Create user
+    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
+    let user_body = serde_json::json!({
+        "secretKey": user_secret,
+        "status": "enabled"
+    });
+    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created user {}", user_name);
+
+    // 2. Add user to group
+    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let add_member_body = serde_json::json!({
+        "group": group_name,
+        "members": [user_name],
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Added {} to group {}", user_name, group_name);
+
+    // 3. Delete the user (backend and cache update so group membership becomes empty)
+    let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
+    awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
+    info!("Deleted user {}", user_name);
+
+    // 4. Deleting the group should succeed (backend has empty members; no stale cache)
+    let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
+    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    info!("Deleted group {} after user was removed", group_name);
+
+    Ok(())
+}

crates/e2e_test/src/lib.rs

@@ -29,6 +29,10 @@ mod data_usage_test;
 #[cfg(test)]
 mod kms;
 
+// Regression test for issue #1797
+#[cfg(test)]
+mod list_objects_duplicates_test;
+
 // Quota tests
 #[cfg(test)]
 mod quota_test;
@@ -36,6 +40,10 @@ mod quota_test;
 #[cfg(test)]
 mod bucket_policy_check_test;
 
+// Regression tests for Issue #2036: anonymous access with PublicAccessBlock
+#[cfg(test)]
+mod anonymous_access_test;
+
 // Special characters in path test modules
 #[cfg(test)]
 mod special_chars_test;
@@ -54,6 +62,10 @@ mod policy;
 
 #[cfg(test)]
 mod compression_test;
+
+// Regression test for Issue #1878: DeleteMarkers not visible immediately after delete_objects
+#[cfg(test)]
+mod delete_objects_versioning_test;
 #[cfg(test)]
 mod protocols;
 
@@ -67,3 +79,7 @@ mod cluster_concurrency_test;
 // PutObject / MultipartUpload with checksum (Content-MD5, x-amz-checksum-*)
 #[cfg(test)]
 mod checksum_upload_test;
+
+// Group deletion tests
+#[cfg(test)]
+mod group_delete_test;

crates/e2e_test/src/list_objects_duplicates_test.rs

@@ -0,0 +1,135 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::primitives::ByteStream;
+    use serial_test::serial;
+    use tracing::info;
+
+    /// Helper function to create an S3 client for testing
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    /// Helper function to create a test bucket
+    async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let mut retries = 20;
+        loop {
+            match client.create_bucket().bucket(bucket).send().await {
+                Ok(_) => {
+                    info!("Bucket {} created successfully", bucket);
+                    return Ok(());
+                }
+                Err(e) => {
+                    // Ignore if bucket already exists
+                    if e.to_string().contains("BucketAlreadyOwnedByYou") || e.to_string().contains("BucketAlreadyExists") {
+                        info!("Bucket {} already exists", bucket);
+                        return Ok(());
+                    }
+                    if retries > 0 {
+                        retries -= 1;
+                        info!("Bucket creation failed, retrying... ({})", retries);
+                        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+                        continue;
+                    }
+                    return Err(Box::new(e));
+                }
+            }
+        }
+    }
+
+    /// Test ensuring that ListObjectsV2 returns unique CommonPrefixes even if "folder" objects exist.
+    ///
+    /// Bug Reference: Issue #1797
+    /// Veeam creates 0-byte objects ending in '/' (e.g. "folder/") to represent folders.
+    /// If "folder/file.txt" also exists, "folder/" is a CommonPrefix.
+    /// The bug was that "folder/" (the object) and "folder/" (derived prefix) were both added to CommonPrefixes
+    /// when delimiter was "/" because the deduplication check was explicitly skipped for "/" delimiter.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_unique_common_prefixes() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 should return unique CommonPrefixes");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-unique-prefixes";
+
+        // Create bucket
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        // 1. Create a file inside a folder
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("folder/file.txt")
+            .body(ByteStream::from_static(b"content"))
+            .send()
+            .await
+            .expect("Failed to create file inside folder");
+
+        // 2. Create the "folder" object itself (Veeam behavior)
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("folder/")
+            .body(ByteStream::from_static(b""))
+            .send()
+            .await
+            .expect("Failed to create folder object");
+
+        // 3. List with delimiter="/"
+        let result = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .delimiter("/")
+            .send()
+            .await
+            .expect("Failed to list objects");
+
+        // Verify prefixes
+        let prefixes = result.common_prefixes();
+        info!("CommonPrefixes: {:?}", prefixes);
+
+        // Should contain "folder/" exactly once
+        let folder_prefixes: Vec<_> = prefixes.iter().filter(|p| p.prefix() == Some("folder/")).collect();
+
+        assert_eq!(
+            folder_prefixes.len(),
+            1,
+            "Expected exactly 1 'folder/' prefix, found {}",
+            folder_prefixes.len()
+        );
+
+        // Verify that "folder/" is NOT returned as an object in Contents.
+        // For this regression test, we expect "folder/" to be represented only as a CommonPrefix
+        // (rolled up from "folder/file.txt" and the explicit "folder/" object), and to appear there
+        // exactly once. It must not appear in Contents at all.
+
+        // Ensure "folder/" is NOT in contents (Contents)
+        let folder_in_contents = result.contents().iter().any(|o| o.key() == Some("folder/"));
+        assert!(
+            !folder_in_contents,
+            "Expected 'folder/' to be rolled up into CommonPrefixes, but found it in Contents"
+        );
+
+        // Stop the RustFS server to ensure proper cleanup
+        env.stop_server();
+    }
+}

crates/e2e_test/src/protocols/README.md

@@ -1,6 +1,6 @@
 # Protocol E2E Tests
 
-FTPS protocol end-to-end tests for RustFS.
+FTPS and WebDAV protocol end-to-end tests for RustFS.
 
 ## Prerequisites
 
@@ -19,11 +19,21 @@ brew install sshpass openssh
 
 ## Running Tests
 
-Run all protocol tests:
+Run all protocol tests (FTPS + WebDAV):
+```bash
+RUSTFS_BUILD_FEATURES=ftps,webdav cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
+```
+
+Run FTPS tests only:
 ```bash
 RUSTFS_BUILD_FEATURES=ftps cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
 ```
 
+Run WebDAV tests only:
+```bash
+RUSTFS_BUILD_FEATURES=webdav cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
+```
+
 ## Test Coverage
 
 ### FTPS Tests
@@ -38,3 +48,13 @@ RUSTFS_BUILD_FEATURES=ftps cargo test --package e2e_test test_protocol_core_suit
 - cdup
 - rmdir delete bucket
 
+### WebDAV Tests
+- PROPFIND at root (list buckets)
+- MKCOL (create bucket)
+- PUT (upload file)
+- GET (download file)
+- PROPFIND on bucket (list objects)
+- DELETE file
+- DELETE bucket
+- Authentication failure test
+

crates/e2e_test/src/protocols/ftps_core.rs

@@ -18,8 +18,7 @@ use crate::common::rustfs_binary_path;
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
 use rcgen::generate_simple_self_signed;
-use rustls::crypto::aws_lc_rs::default_provider;
-use rustls::{ClientConfig, RootCertStore};
+use rustls::{ClientConfig, RootCertStore, pki_types::CertificateDer, pki_types::pem::PemObject};
 use std::io::Cursor;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -74,8 +73,8 @@ pub async fn test_ftps_core_operations() -> Result<()> {
             .await
             .map_err(|e| anyhow::anyhow!("{}", e))?;
 
-        // Install the aws-lc-rs crypto provider
-        default_provider()
+        // Build ServerConfig with SNI support
+        rustls::crypto::aws_lc_rs::default_provider()
             .install_default()
             .map_err(|e| anyhow::anyhow!("Failed to install crypto provider: {:?}", e))?;
 
@@ -84,7 +83,7 @@ pub async fn test_ftps_core_operations() -> Result<()> {
         // Add the self-signed certificate to the trust store for e2e
         // Note: In a real environment, you'd use proper root certificates
         let cert_pem = default_cert.cert.pem();
-        let cert_der = rustls_pemfile::certs(&mut Cursor::new(cert_pem))
+        let cert_der = CertificateDer::pem_reader_iter(&mut Cursor::new(cert_pem))
             .collect::<Result<Vec<_>, _>>()
             .map_err(|e| anyhow::anyhow!("Failed to parse cert: {}", e))?;
 

crates/e2e_test/src/protocols/mod.rs

@@ -12,8 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Protocol tests for FTPS
+//! Protocol tests for FTPS and WebDAV
 
 pub mod ftps_core;
 pub mod test_env;
 pub mod test_runner;
+pub mod webdav_core;

crates/e2e_test/src/protocols/test_runner.rs

@@ -16,6 +16,7 @@
 
 use crate::common::init_logging;
 use crate::protocols::ftps_core::test_ftps_core_operations;
+use crate::protocols::webdav_core::test_webdav_core_operations;
 use std::time::Instant;
 use tokio::time::{Duration, sleep};
 use tracing::{error, info};
@@ -59,9 +60,14 @@ struct TestDefinition {
 impl ProtocolTestSuite {
     /// Create default test suite
     pub fn new() -> Self {
-        let tests = vec![TestDefinition {
-            name: "test_ftps_core_operations".to_string(),
-        }];
+        let tests = vec![
+            TestDefinition {
+                name: "test_ftps_core_operations".to_string(),
+            },
+            TestDefinition {
+                name: "test_webdav_core_operations".to_string(),
+            },
+        ];
 
         Self { tests }
     }
@@ -83,6 +89,10 @@ impl ProtocolTestSuite {
                     info!("=== Starting FTPS Module Test ===");
                     "FTPS core operations (put, ls, mkdir, rmdir, delete)"
                 }
+                "test_webdav_core_operations" => {
+                    info!("=== Starting WebDAV Core Test ===");
+                    "WebDAV core operations (MKCOL, PUT, GET, DELETE, PROPFIND)"
+                }
                 _ => "",
             };
 
@@ -121,6 +131,7 @@ impl ProtocolTestSuite {
     async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         match test_def.name.as_str() {
             "test_ftps_core_operations" => test_ftps_core_operations().await.map_err(|e| e.into()),
+            "test_webdav_core_operations" => test_webdav_core_operations().await.map_err(|e| e.into()),
             _ => Err(format!("Test {} not implemented", test_def.name).into()),
         }
     }

crates/e2e_test/src/protocols/webdav_core.rs

@@ -0,0 +1,207 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Core WebDAV tests
+
+use crate::common::rustfs_binary_path;
+use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
+use anyhow::Result;
+use base64::Engine;
+use reqwest::Client;
+use tokio::process::Command;
+use tracing::info;
+
+// Fixed WebDAV port for testing
+const WEBDAV_PORT: u16 = 9080;
+const WEBDAV_ADDRESS: &str = "127.0.0.1:9080";
+
+/// Create HTTP client with basic auth
+fn create_client() -> Client {
+    Client::builder()
+        .danger_accept_invalid_certs(true)
+        .build()
+        .expect("Failed to create HTTP client")
+}
+
+/// Get basic auth header value
+fn basic_auth_header() -> String {
+    let credentials = format!("{}:{}", DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY);
+    let encoded = base64::engine::general_purpose::STANDARD.encode(credentials);
+    format!("Basic {}", encoded)
+}
+
+/// Test WebDAV: MKCOL (create bucket), PUT, GET, DELETE, PROPFIND operations
+pub async fn test_webdav_core_operations() -> Result<()> {
+    let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;
+
+    // Start server manually
+    info!("Starting WebDAV server on {}", WEBDAV_ADDRESS);
+    let binary_path = rustfs_binary_path();
+    let mut server_process = Command::new(&binary_path)
+        .env("RUSTFS_WEBDAV_ENABLE", "true")
+        .env("RUSTFS_WEBDAV_ADDRESS", WEBDAV_ADDRESS)
+        .env("RUSTFS_WEBDAV_TLS_ENABLED", "false") // No TLS for testing
+        .arg(&env.temp_dir)
+        .spawn()?;
+
+    // Ensure server is cleaned up even on failure
+    let result = async {
+        // Wait for server to be ready
+        ProtocolTestEnvironment::wait_for_port_ready(WEBDAV_PORT, 30)
+            .await
+            .map_err(|e| anyhow::anyhow!("{}", e))?;
+
+        let client = create_client();
+        let auth_header = basic_auth_header();
+        let base_url = format!("http://{}", WEBDAV_ADDRESS);
+
+        // Test PROPFIND at root (list buckets)
+        info!("Testing WebDAV: PROPFIND at root (list buckets)");
+        let resp = client
+            .request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), &base_url)
+            .header("Authorization", &auth_header)
+            .header("Depth", "1")
+            .send()
+            .await?;
+        assert!(
+            resp.status().is_success() || resp.status().as_u16() == 207,
+            "PROPFIND at root should succeed, got: {}",
+            resp.status()
+        );
+        info!("PASS: PROPFIND at root successful");
+
+        // Test MKCOL (create bucket)
+        let bucket_name = "webdav-test-bucket";
+        info!("Testing WebDAV: MKCOL (create bucket '{}')", bucket_name);
+        let resp = client
+            .request(reqwest::Method::from_bytes(b"MKCOL").unwrap(), format!("{}/{}", base_url, bucket_name))
+            .header("Authorization", &auth_header)
+            .send()
+            .await?;
+        assert!(
+            resp.status().is_success() || resp.status().as_u16() == 201,
+            "MKCOL should succeed, got: {}",
+            resp.status()
+        );
+        info!("PASS: MKCOL bucket '{}' successful", bucket_name);
+
+        // Test PUT (upload file)
+        let filename = "test-file.txt";
+        let file_content = "Hello, WebDAV!";
+        info!("Testing WebDAV: PUT (upload file '{}')", filename);
+        let resp = client
+            .put(format!("{}/{}/{}", base_url, bucket_name, filename))
+            .header("Authorization", &auth_header)
+            .body(file_content)
+            .send()
+            .await?;
+        assert!(
+            resp.status().is_success() || resp.status().as_u16() == 201,
+            "PUT should succeed, got: {}",
+            resp.status()
+        );
+        info!("PASS: PUT file '{}' successful", filename);
+
+        // Test GET (download file)
+        info!("Testing WebDAV: GET (download file '{}')", filename);
+        let resp = client
+            .get(format!("{}/{}/{}", base_url, bucket_name, filename))
+            .header("Authorization", &auth_header)
+            .send()
+            .await?;
+        assert!(resp.status().is_success(), "GET should succeed, got: {}", resp.status());
+        let downloaded_content = resp.text().await?;
+        assert_eq!(downloaded_content, file_content, "Downloaded content should match uploaded content");
+        info!("PASS: GET file '{}' successful, content matches", filename);
+
+        // Test PROPFIND on bucket (list objects)
+        info!("Testing WebDAV: PROPFIND on bucket (list objects)");
+        let resp = client
+            .request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), format!("{}/{}", base_url, bucket_name))
+            .header("Authorization", &auth_header)
+            .header("Depth", "1")
+            .send()
+            .await?;
+        assert!(
+            resp.status().is_success() || resp.status().as_u16() == 207,
+            "PROPFIND on bucket should succeed, got: {}",
+            resp.status()
+        );
+        let body = resp.text().await?;
+        assert!(body.contains(filename), "File should appear in PROPFIND response");
+        info!("PASS: PROPFIND on bucket successful, file '{}' found", filename);
+
+        // Test DELETE file
+        info!("Testing WebDAV: DELETE file '{}'", filename);
+        let resp = client
+            .delete(format!("{}/{}/{}", base_url, bucket_name, filename))
+            .header("Authorization", &auth_header)
+            .send()
+            .await?;
+        assert!(
+            resp.status().is_success() || resp.status().as_u16() == 204,
+            "DELETE file should succeed, got: {}",
+            resp.status()
+        );
+        info!("PASS: DELETE file '{}' successful", filename);
+
+        // Verify file is deleted
+        info!("Testing WebDAV: Verify file is deleted");
+        let resp = client
+            .get(format!("{}/{}/{}", base_url, bucket_name, filename))
+            .header("Authorization", &auth_header)
+            .send()
+            .await?;
+        assert!(
+            resp.status().as_u16() == 404,
+            "GET deleted file should return 404, got: {}",
+            resp.status()
+        );
+        info!("PASS: Verified file '{}' is deleted", filename);
+
+        // Test DELETE bucket
+        info!("Testing WebDAV: DELETE bucket '{}'", bucket_name);
+        let resp = client
+            .delete(format!("{}/{}", base_url, bucket_name))
+            .header("Authorization", &auth_header)
+            .send()
+            .await?;
+        assert!(
+            resp.status().is_success() || resp.status().as_u16() == 204,
+            "DELETE bucket should succeed, got: {}",
+            resp.status()
+        );
+        info!("PASS: DELETE bucket '{}' successful", bucket_name);
+
+        // Test authentication failure
+        info!("Testing WebDAV: Authentication failure");
+        let resp = client
+            .request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), &base_url)
+            .header("Authorization", "Basic aW52YWxpZDppbnZhbGlk") // invalid:invalid
+            .send()
+            .await?;
+        assert_eq!(resp.status().as_u16(), 401, "Invalid auth should return 401, got: {}", resp.status());
+        info!("PASS: Authentication failure test successful");
+
+        info!("WebDAV core tests passed");
+        Ok(())
+    }
+    .await;
+
+    // Always cleanup server process
+    let _ = server_process.kill().await;
+    let _ = server_process.wait().await;
+
+    result
+}

crates/ecstore/AGENTS.md

@@ -0,0 +1,28 @@
+# ECStore Crate Instructions
+
+Applies to `crates/ecstore/`.
+
+## Data Durability and Integrity
+
+- Do not weaken quorum checks, bitrot checks, or metadata validation paths.
+- Treat any change affecting read/write/repair correctness as high risk and test accordingly.
+- Prefer explicit failure over silent data corruption or implicit success.
+
+## Performance-Critical Paths
+
+- Be careful with allocations and locking in hot paths.
+- Keep network and disk operations async-friendly; avoid introducing unnecessary blocking.
+- Benchmark-sensitive changes should include measurable rationale.
+
+## Cross-Module Coordination
+
+- Validate behavior impacts on:
+  - `rustfs/src/storage/`
+  - `crates/filemeta/`
+  - `crates/heal/`
+  - `crates/checksums/`
+
+## Suggested Validation
+
+- `cargo test -p rustfs-ecstore`
+- Full gate before commit: `make pre-commit`

crates/ecstore/Cargo.toml

@@ -44,6 +44,7 @@ rustfs-credentials = { workspace = true }
 rustfs-common.workspace = true
 rustfs-policy.workspace = true
 rustfs-protos.workspace = true
+rustfs-s3-common = { workspace = true }
 async-trait.workspace = true
 bytes.workspace = true
 byteorder = { workspace = true }
@@ -111,6 +112,8 @@ google-cloud-storage = { workspace = true }
 google-cloud-auth = { workspace = true }
 aws-config = { workspace = true }
 faster-hex = { workspace = true }
+ratelimit = { workspace = true }
+aws-smithy-http-client.workspace = true
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

crates/ecstore/src/bitrot.rs

@@ -39,6 +39,7 @@ pub async fn create_bitrot_reader(
     length: usize,
     shard_size: usize,
     checksum_algo: HashAlgorithm,
+    skip_verify: bool,
 ) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
     // Calculate the total length to read, including the checksum overhead
     let length = length.div_ceil(shard_size) * checksum_algo.size() + length;
@@ -47,13 +48,18 @@ pub async fn create_bitrot_reader(
         // Use inline data
         let mut rd = Cursor::new(data.to_vec());
         rd.set_position(offset as u64);
-        let reader = BitrotReader::new(Box::new(rd) as Box<dyn AsyncRead + Send + Sync + Unpin>, shard_size, checksum_algo);
+        let reader = BitrotReader::new(
+            Box::new(rd) as Box<dyn AsyncRead + Send + Sync + Unpin>,
+            shard_size,
+            checksum_algo,
+            skip_verify,
+        );
         Ok(Some(reader))
     } else if let Some(disk) = disk {
         // Read from disk
         match disk.read_file_stream(bucket, path, offset, length - offset).await {
             Ok(rd) => {
-                let reader = BitrotReader::new(rd, shard_size, checksum_algo);
+                let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
                 Ok(Some(reader))
             }
             Err(e) => Err(e),
@@ -116,7 +122,7 @@ mod tests {
         let checksum_algo = HashAlgorithm::HighwayHash256;
 
         let result =
-            create_bitrot_reader(Some(test_data), None, "test-bucket", "test-path", 0, 0, shard_size, checksum_algo).await;
+            create_bitrot_reader(Some(test_data), None, "test-bucket", "test-path", 0, 0, shard_size, checksum_algo, false).await;
 
         assert!(result.is_ok());
         assert!(result.unwrap().is_some());
@@ -127,7 +133,8 @@ mod tests {
         let shard_size = 16;
         let checksum_algo = HashAlgorithm::HighwayHash256;
 
-        let result = create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo).await;
+        let result =
+            create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo, false).await;
 
         assert!(result.is_ok());
         assert!(result.unwrap().is_none());

crates/ecstore/src/bucket/bandwidth/mod.rs

@@ -0,0 +1,16 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod monitor;
+pub mod reader;

crates/ecstore/src/bucket/bandwidth/monitor.rs

@@ -0,0 +1,589 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::bucket::bandwidth::reader::BucketOptions;
+use ratelimit::{Error as RatelimitError, Ratelimiter};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex, RwLock};
+use std::time::{Duration, Instant};
+use tracing::warn;
+
+/// BETA_BUCKET is the weight used to calculate exponential moving average
+const BETA_BUCKET: f64 = 0.1;
+
+#[derive(Clone)]
+pub struct BucketThrottle {
+    limiter: Arc<Mutex<Ratelimiter>>,
+    pub node_bandwidth_per_sec: i64,
+}
+
+impl BucketThrottle {
+    fn new(node_bandwidth_per_sec: i64) -> Result<Self, RatelimitError> {
+        let node_bandwidth_per_sec = node_bandwidth_per_sec.max(1);
+        let amount = node_bandwidth_per_sec as u64;
+        let limiter_inner = Ratelimiter::builder(amount, Duration::from_secs(1))
+            .max_tokens(amount)
+            .build()?;
+        Ok(Self {
+            limiter: Arc::new(Mutex::new(limiter_inner)),
+            node_bandwidth_per_sec,
+        })
+    }
+
+    pub fn burst(&self) -> u64 {
+        self.limiter.lock().unwrap_or_else(|e| e.into_inner()).max_tokens()
+    }
+
+    /// The ratelimit crate (0.10.0) does not provide a bulk token consumption API.
+    /// try_wait() first to consume 1 token AND trigger the internal refill
+    /// mechanism (tokens are only refilled during try_wait/wait calls).
+    /// directly adjust available tokens via set_available() to consume the remaining amount.
+    pub(crate) fn consume(&self, n: u64) -> (u64, f64, u64) {
+        let guard = self.limiter.lock().unwrap_or_else(|e| {
+            warn!("bucket throttle mutex poisoned, recovering");
+            e.into_inner()
+        });
+        if n == 0 {
+            return (0, guard.rate(), 0);
+        }
+        let mut consumed = 0u64;
+        if guard.try_wait().is_ok() {
+            consumed = 1;
+        }
+        let available = guard.available();
+        let to_consume = n - consumed;
+        let batch = to_consume.min(available);
+        if batch > 0 {
+            let _ = guard.set_available(available - batch);
+            consumed += batch;
+        }
+        let deficit = n.saturating_sub(consumed);
+        let rate = guard.rate();
+        (deficit, rate, consumed)
+    }
+}
+
+#[derive(Debug)]
+pub struct BucketMeasurement {
+    bytes_since_last_window: AtomicU64,
+    start_time: Mutex<Option<Instant>>,
+    exp_moving_avg: Mutex<f64>,
+}
+
+impl BucketMeasurement {
+    pub fn new(init_time: Instant) -> Self {
+        Self {
+            bytes_since_last_window: AtomicU64::new(0),
+            start_time: Mutex::new(Some(init_time)),
+            exp_moving_avg: Mutex::new(0.0),
+        }
+    }
+
+    pub fn increment_bytes(&self, bytes: u64) {
+        self.bytes_since_last_window.fetch_add(bytes, Ordering::Relaxed);
+    }
+
+    pub fn update_exponential_moving_average(&self, end_time: Instant) {
+        let mut start_time = self.start_time.lock().unwrap_or_else(|e| {
+            warn!("bucket measurement start_time mutex poisoned, recovering");
+            e.into_inner()
+        });
+        let previous_start = *start_time;
+        *start_time = Some(end_time);
+
+        let Some(prev_start) = previous_start else {
+            return;
+        };
+
+        if prev_start > end_time {
+            return;
+        }
+
+        let duration = end_time.duration_since(prev_start);
+        if duration.is_zero() {
+            return;
+        }
+
+        let bytes_since_last_window = self.bytes_since_last_window.swap(0, Ordering::Relaxed);
+        let increment = bytes_since_last_window as f64 / duration.as_secs_f64();
+
+        let mut exp_moving_avg = self.exp_moving_avg.lock().unwrap_or_else(|e| {
+            warn!("bucket measurement exp_moving_avg mutex poisoned, recovering");
+            e.into_inner()
+        });
+
+        if *exp_moving_avg == 0.0 {
+            *exp_moving_avg = increment;
+            return;
+        }
+
+        *exp_moving_avg = exponential_moving_average(BETA_BUCKET, *exp_moving_avg, increment);
+    }
+
+    pub fn get_exp_moving_avg_bytes_per_second(&self) -> f64 {
+        *self.exp_moving_avg.lock().unwrap_or_else(|e| {
+            warn!("bucket measurement exp_moving_avg mutex poisoned, recovering");
+            e.into_inner()
+        })
+    }
+}
+
+fn exponential_moving_average(beta: f64, previous_avg: f64, increment_avg: f64) -> f64 {
+    (1f64 - beta) * increment_avg + beta * previous_avg
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BandwidthDetails {
+    pub limit_bytes_per_sec: i64,
+    pub current_bandwidth_bytes_per_sec: f64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BucketBandwidthReport {
+    pub bucket_stats: HashMap<BucketOptions, BandwidthDetails>,
+}
+
+pub struct Monitor {
+    t_lock: RwLock<HashMap<BucketOptions, BucketThrottle>>,
+    m_lock: RwLock<HashMap<BucketOptions, BucketMeasurement>>,
+    pub node_count: u64,
+}
+
+impl Monitor {
+    pub fn new(num_nodes: u64) -> Arc<Self> {
+        let node_cnt = num_nodes.max(1);
+        let m = Arc::new(Monitor {
+            t_lock: RwLock::new(HashMap::new()),
+            m_lock: RwLock::new(HashMap::new()),
+            node_count: node_cnt,
+        });
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            let weak = Arc::downgrade(&m);
+            handle.spawn(async move {
+                let mut interval = tokio::time::interval(Duration::from_secs(2));
+                loop {
+                    interval.tick().await;
+                    let Some(monitor) = weak.upgrade() else { break };
+                    monitor.update_moving_avg();
+                }
+            });
+        }
+        m
+    }
+
+    pub fn init_measurement(&self, opts: &BucketOptions) {
+        let mut guard = self.m_lock.write().unwrap_or_else(|e| {
+            warn!("bucket monitor measurement rwlock write poisoned, recovering");
+            e.into_inner()
+        });
+        guard
+            .entry(opts.clone())
+            .or_insert_with(|| BucketMeasurement::new(Instant::now()));
+    }
+
+    pub fn update_measurement(&self, opts: &BucketOptions, bytes: u64) {
+        {
+            let guard = self.m_lock.read().unwrap_or_else(|e| {
+                warn!("bucket monitor measurement rwlock read poisoned, recovering");
+                e.into_inner()
+            });
+            if let Some(measurement) = guard.get(opts) {
+                measurement.increment_bytes(bytes);
+                return;
+            }
+        }
+
+        // Miss path: write lock + insert once, then increment.
+        let mut guard = self.m_lock.write().unwrap_or_else(|e| {
+            warn!("bucket monitor measurement rwlock write poisoned, recovering");
+            e.into_inner()
+        });
+
+        // Double-check after lock upgrade in case another thread inserted it.
+        let measurement = guard
+            .entry(opts.clone())
+            .or_insert_with(|| BucketMeasurement::new(Instant::now()));
+
+        measurement.increment_bytes(bytes);
+    }
+
+    pub fn update_moving_avg(&self) {
+        let now = Instant::now();
+        let guard = self.m_lock.read().unwrap_or_else(|e| {
+            warn!("bucket monitor measurement rwlock read poisoned, recovering");
+            e.into_inner()
+        });
+        for measurement in guard.values() {
+            measurement.update_exponential_moving_average(now);
+        }
+    }
+
+    pub fn get_report(&self, select_bucket: impl Fn(&str) -> bool) -> BucketBandwidthReport {
+        let t_guard = self.t_lock.read().unwrap_or_else(|e| {
+            warn!("bucket monitor throttle rwlock read poisoned, recovering");
+            e.into_inner()
+        });
+        let m_guard = self.m_lock.read().unwrap_or_else(|e| {
+            warn!("bucket monitor measurement rwlock read poisoned, recovering");
+            e.into_inner()
+        });
+        let mut bucket_stats = HashMap::new();
+        for (opts, throttle) in t_guard.iter() {
+            if !select_bucket(&opts.name) {
+                continue;
+            }
+            let mut current_bandwidth_bytes_per_sec = 0.0;
+            if let Some(measurement) = m_guard.get(opts) {
+                current_bandwidth_bytes_per_sec = measurement.get_exp_moving_avg_bytes_per_second();
+            }
+            bucket_stats.insert(
+                opts.clone(),
+                BandwidthDetails {
+                    limit_bytes_per_sec: throttle.node_bandwidth_per_sec * self.node_count as i64,
+                    current_bandwidth_bytes_per_sec,
+                },
+            );
+        }
+        BucketBandwidthReport { bucket_stats }
+    }
+
+    pub fn delete_bucket(&self, bucket: &str) {
+        self.t_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor throttle rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .retain(|opts, _| opts.name != bucket);
+        self.m_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor measurement rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .retain(|opts, _| opts.name != bucket);
+    }
+
+    pub fn delete_bucket_throttle(&self, bucket: &str, arn: &str) {
+        let opts = BucketOptions {
+            name: bucket.to_string(),
+            replication_arn: arn.to_string(),
+        };
+        self.t_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor throttle rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .remove(&opts);
+        self.m_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor measurement rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .remove(&opts);
+    }
+
+    pub fn throttle(&self, opts: &BucketOptions) -> Option<BucketThrottle> {
+        self.t_lock
+            .read()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor throttle rwlock read poisoned, recovering");
+                e.into_inner()
+            })
+            .get(opts)
+            .cloned()
+    }
+
+    pub fn set_bandwidth_limit(&self, bucket: &str, arn: &str, limit: i64) {
+        if limit <= 0 {
+            warn!(
+                bucket = bucket,
+                arn = arn,
+                limit = limit,
+                "invalid bandwidth limit, must be positive; ignoring"
+            );
+            return;
+        }
+        let limit_bytes = limit / self.node_count as i64;
+        if limit_bytes == 0 && limit > 0 {
+            warn!(
+                bucket = bucket,
+                arn = arn,
+                limit = limit,
+                node_count = self.node_count,
+                "bandwidth limit too small for cluster size, per-node limit will clamp to 1 byte/s"
+            );
+        }
+        let opts = BucketOptions {
+            name: bucket.to_string(),
+            replication_arn: arn.to_string(),
+        };
+        let throttle = match BucketThrottle::new(limit_bytes) {
+            Ok(t) => t,
+            Err(e) => {
+                warn!(
+                    bucket = bucket,
+                    arn = arn,
+                    limit_bytes = limit_bytes,
+                    err = %e,
+                    "failed to build bandwidth throttle, throttling disabled for this target"
+                );
+                return;
+            }
+        };
+        self.t_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor throttle rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .insert(opts, throttle);
+    }
+
+    pub fn is_throttled(&self, bucket: &str, arn: &str) -> bool {
+        let opt = BucketOptions {
+            name: bucket.to_string(),
+            replication_arn: arn.to_string(),
+        };
+        self.t_lock
+            .read()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor throttle rwlock read poisoned, recovering");
+                e.into_inner()
+            })
+            .contains_key(&opt)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::panic::{AssertUnwindSafe, catch_unwind};
+
+    #[test]
+    fn test_set_and_get_throttle_with_node_split() {
+        let monitor = Monitor::new(4);
+        monitor.set_bandwidth_limit("b1", "arn1", 400);
+
+        let throttle = monitor
+            .throttle(&BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            })
+            .expect("throttle should exist");
+
+        assert_eq!(throttle.node_bandwidth_per_sec, 100);
+        assert!(monitor.is_throttled("b1", "arn1"));
+    }
+
+    #[test]
+    fn test_delete_bucket_throttle() {
+        let monitor = Monitor::new(2);
+        monitor.set_bandwidth_limit("b1", "arn1", 200);
+        assert!(monitor.is_throttled("b1", "arn1"));
+
+        monitor.delete_bucket_throttle("b1", "arn1");
+        assert!(!monitor.is_throttled("b1", "arn1"));
+    }
+
+    #[test]
+    fn test_delete_bucket_removes_all_arns() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 100);
+        monitor.set_bandwidth_limit("b1", "arn2", 100);
+        monitor.set_bandwidth_limit("b2", "arn3", 100);
+
+        monitor.delete_bucket("b1");
+
+        assert!(!monitor.is_throttled("b1", "arn1"));
+        assert!(!monitor.is_throttled("b1", "arn2"));
+        assert!(monitor.is_throttled("b2", "arn3"));
+    }
+
+    #[test]
+    fn test_set_bandwidth_limit_ignores_non_positive() {
+        let monitor = Monitor::new(2);
+
+        monitor.set_bandwidth_limit("b1", "arn1", 0);
+        assert!(!monitor.is_throttled("b1", "arn1"));
+
+        monitor.set_bandwidth_limit("b1", "arn1", -10);
+        assert!(!monitor.is_throttled("b1", "arn1"));
+    }
+
+    #[test]
+    fn test_consume_returns_deficit_when_tokens_exhausted() {
+        let throttle = BucketThrottle::new(100).expect("test");
+
+        let (deficit, rate, _consumed) = throttle.consume(200);
+
+        assert!(deficit > 0);
+        assert!(rate > 0.0);
+    }
+
+    #[test]
+    fn test_consume_no_deficit_when_tokens_sufficient() {
+        let throttle = BucketThrottle::new(10000).expect("test");
+
+        std::thread::sleep(std::time::Duration::from_millis(1100));
+
+        let (deficit, _rate, _consumed) = throttle.consume(5000);
+        assert_eq!(deficit, 0);
+    }
+
+    #[test]
+    fn test_burst_equals_bandwidth() {
+        let throttle = BucketThrottle::new(500).expect("test");
+        assert_eq!(throttle.burst(), 500);
+    }
+
+    #[test]
+    fn test_concurrent_consume() {
+        let throttle = BucketThrottle::new(10000).expect("test");
+        std::thread::sleep(std::time::Duration::from_millis(1100));
+
+        let mut handles = vec![];
+        for _ in 0..10 {
+            let t = throttle.clone();
+            handles.push(std::thread::spawn(move || t.consume(100)));
+        }
+
+        let mut total_deficit = 0u64;
+        let mut total_consumed = 0u64;
+        for h in handles {
+            let (deficit, _rate, consumed) = h.join().unwrap();
+            total_consumed += consumed;
+            total_deficit += deficit;
+        }
+
+        assert_eq!(total_consumed + total_deficit, 1000);
+        assert!(total_consumed <= 10000);
+    }
+
+    #[test]
+    fn test_zero_bandwidth_clamped_to_one() {
+        let throttle = BucketThrottle::new(0).expect("test");
+        assert_eq!(throttle.burst(), 1);
+        assert_eq!(throttle.node_bandwidth_per_sec, 1);
+    }
+
+    #[test]
+    fn test_negative_bandwidth_clamped_to_one() {
+        let throttle = BucketThrottle::new(-100).expect("test");
+        assert_eq!(throttle.burst(), 1);
+        assert_eq!(throttle.node_bandwidth_per_sec, 1);
+    }
+
+    #[test]
+    fn test_update_bandwidth_limit_overrides_previous() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 1000);
+
+        let opts = BucketOptions {
+            name: "b1".to_string(),
+            replication_arn: "arn1".to_string(),
+        };
+        let t1 = monitor.throttle(&opts).expect("throttle should exist");
+        assert_eq!(t1.burst(), 1000);
+        assert_eq!(t1.node_bandwidth_per_sec, 1000);
+
+        monitor.set_bandwidth_limit("b1", "arn1", 500);
+
+        let t2 = monitor.throttle(&opts).expect("throttle should exist after update");
+        assert_eq!(t2.burst(), 500);
+        assert_eq!(t2.node_bandwidth_per_sec, 500);
+    }
+
+    #[test]
+    fn test_bucket_measurement_recovers_from_poisoned_mutexes() {
+        let measurement = BucketMeasurement::new(Instant::now());
+
+        let _ = catch_unwind(AssertUnwindSafe(|| {
+            let _guard = measurement.start_time.lock().unwrap();
+            panic!("poison start_time mutex");
+        }));
+        measurement.increment_bytes(64);
+        measurement.update_exponential_moving_average(Instant::now() + Duration::from_secs(1));
+
+        let _ = catch_unwind(AssertUnwindSafe(|| {
+            let _guard = measurement.exp_moving_avg.lock().unwrap();
+            panic!("poison exp_moving_avg mutex");
+        }));
+        measurement.increment_bytes(32);
+        measurement.update_exponential_moving_average(Instant::now() + Duration::from_secs(2));
+
+        let value = measurement.get_exp_moving_avg_bytes_per_second();
+        assert!(value.is_finite());
+        assert!(value >= 0.0);
+    }
+
+    #[test]
+    fn test_get_report_limit_and_current_bandwidth_after_measurement() {
+        let monitor = Monitor::new(4);
+        monitor.set_bandwidth_limit("b1", "arn1", 400);
+        let opts = BucketOptions {
+            name: "b1".to_string(),
+            replication_arn: "arn1".to_string(),
+        };
+        monitor.init_measurement(&opts);
+        monitor.update_measurement(&opts, 500);
+        monitor.update_measurement(&opts, 500);
+        std::thread::sleep(Duration::from_millis(110));
+        monitor.update_moving_avg();
+
+        let report = monitor.get_report(|name| name == "b1");
+        let details = report.bucket_stats.get(&opts).expect("report should contain b1/arn1");
+        assert_eq!(details.limit_bytes_per_sec, 400);
+        assert!(
+            details.current_bandwidth_bytes_per_sec > 0.0,
+            "current_bandwidth should be positive after update_measurement and update_moving_avg"
+        );
+        assert!(
+            details.current_bandwidth_bytes_per_sec < 20000.0,
+            "current_bandwidth should be in reasonable range"
+        );
+    }
+
+    #[test]
+    fn test_get_report_select_bucket_filters() {
+        let monitor = Monitor::new(2);
+        monitor.set_bandwidth_limit("b1", "arn1", 100);
+        monitor.set_bandwidth_limit("b2", "arn2", 200);
+        let opts_b1 = BucketOptions {
+            name: "b1".to_string(),
+            replication_arn: "arn1".to_string(),
+        };
+        let opts_b2 = BucketOptions {
+            name: "b2".to_string(),
+            replication_arn: "arn2".to_string(),
+        };
+        monitor.init_measurement(&opts_b1);
+        monitor.init_measurement(&opts_b2);
+
+        let report_all = monitor.get_report(|_| true);
+        assert_eq!(report_all.bucket_stats.len(), 2);
+
+        let report_b1 = monitor.get_report(|name| name == "b1");
+        assert_eq!(report_b1.bucket_stats.len(), 1);
+        assert_eq!(report_b1.bucket_stats.get(&opts_b1).unwrap().limit_bytes_per_sec, 100);
+
+        let report_b2 = monitor.get_report(|name| name == "b2");
+        assert_eq!(report_b2.bucket_stats.len(), 1);
+        assert_eq!(report_b2.bucket_stats.get(&opts_b2).unwrap().limit_bytes_per_sec, 200);
+    }
+}

crates/ecstore/src/bucket/bandwidth/reader.rs

@@ -0,0 +1,348 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::bucket::bandwidth::monitor::Monitor;
+use serde::{Deserialize, Serialize};
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+use tokio::io::{AsyncRead, ReadBuf};
+use tokio::time::Sleep;
+use tracing::{debug, warn};
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct BucketOptions {
+    pub name: String,
+    pub replication_arn: String,
+}
+
+pub struct MonitorReaderOptions {
+    pub bucket_options: BucketOptions,
+    pub header_size: usize,
+}
+
+struct WaitState {
+    sleep: Pin<Box<Sleep>>,
+}
+
+pub struct MonitoredReader<R> {
+    r: R,
+    m: Arc<Monitor>,
+    opts: MonitorReaderOptions,
+    wait_state: std::sync::Mutex<Option<WaitState>>,
+    temp_buf: Vec<u8>,
+}
+
+impl<R> MonitoredReader<R> {
+    pub fn new(m: Arc<Monitor>, r: R, opts: MonitorReaderOptions) -> Self {
+        let throttle = m.throttle(&opts.bucket_options);
+        debug!(
+            bucket = opts.bucket_options.name,
+            arn = opts.bucket_options.replication_arn,
+            throttle_active = throttle.is_some(),
+            limit_bps = throttle.as_ref().map(|t| t.node_bandwidth_per_sec).unwrap_or(0),
+            header_size = opts.header_size,
+            "MonitoredReader created"
+        );
+        if throttle.is_some() {
+            m.init_measurement(&opts.bucket_options);
+        }
+        MonitoredReader {
+            r,
+            m,
+            opts,
+            wait_state: std::sync::Mutex::new(None),
+            temp_buf: Vec::new(),
+        }
+    }
+}
+
+impl<R: AsyncRead + Unpin> AsyncRead for MonitoredReader<R> {
+    fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+        let this = self.get_mut();
+
+        {
+            let mut guard = this.wait_state.lock().unwrap_or_else(|e| {
+                warn!("MonitoredReader wait_state mutex poisoned, recovering");
+                e.into_inner()
+            });
+            if let Some(ref mut ws) = *guard {
+                match ws.sleep.as_mut().poll(cx) {
+                    Poll::Pending => return Poll::Pending,
+                    Poll::Ready(()) => {
+                        *guard = None;
+                        drop(guard);
+                    }
+                }
+            }
+        }
+
+        let throttle = match this.m.throttle(&this.opts.bucket_options) {
+            Some(t) => t,
+            None => return Pin::new(&mut this.r).poll_read(cx, buf),
+        };
+
+        let b = throttle.burst();
+        debug_assert!(b >= 1, "burst must be at least 1");
+        let (need, tokens) = calc_need_and_tokens(b, buf.remaining(), &mut this.opts.header_size);
+        let (deficit, rate, consumed) = throttle.consume(tokens);
+        let need = need.min(consumed as usize);
+
+        if deficit > 0 && rate > 0.0 {
+            let duration = std::time::Duration::from_secs_f64(deficit as f64 / rate);
+            debug!(
+                tokens = tokens,
+                deficit = deficit,
+                rate = rate,
+                sleep_ms = duration.as_millis() as u64,
+                "bandwidth throttle sleep"
+            );
+            let mut sleep = Box::pin(tokio::time::sleep(duration));
+            match sleep.as_mut().poll(cx) {
+                Poll::Pending => {
+                    *this.wait_state.lock().unwrap_or_else(|e| {
+                        warn!("MonitoredReader wait_state mutex poisoned, recovering");
+                        e.into_inner()
+                    }) = Some(WaitState { sleep });
+                    return Poll::Pending;
+                }
+                Poll::Ready(()) => {}
+            }
+        }
+
+        let filled_before = buf.filled().len();
+        let result = poll_limited_read(&mut this.r, cx, buf, need, &mut this.temp_buf);
+        if let Poll::Ready(Ok(())) = result {
+            let read_bytes = buf.filled().len().saturating_sub(filled_before) as u64;
+            if read_bytes > 0 {
+                this.m.update_measurement(&this.opts.bucket_options, read_bytes);
+            }
+        }
+        result
+    }
+}
+
+fn calc_need_and_tokens(burst: u64, need_upper: usize, header_size: &mut usize) -> (usize, u64) {
+    let hdr = *header_size;
+    let mut need = need_upper;
+    let tokens: u64 = if hdr > 0 {
+        if (hdr as u64) < burst {
+            *header_size = 0;
+            need = ((burst - hdr as u64) as usize).min(need);
+            need as u64 + hdr as u64
+        } else {
+            *header_size -= burst as usize;
+            need = 0;
+            burst
+        }
+    } else {
+        need = need.min(burst as usize);
+        need as u64
+    };
+    (need, tokens)
+}
+
+fn poll_limited_read<R: AsyncRead + Unpin>(
+    r: &mut R,
+    cx: &mut Context<'_>,
+    buf: &mut ReadBuf<'_>,
+    limit: usize,
+    reusable_buf: &mut Vec<u8>,
+) -> Poll<std::io::Result<()>> {
+    let remaining = buf.remaining();
+    if limit == 0 || remaining == 0 {
+        return Poll::Ready(Ok(()));
+    }
+
+    if remaining <= limit {
+        return Pin::new(r).poll_read(cx, buf);
+    }
+
+    reusable_buf.resize(limit, 0);
+    let mut temp_buf = ReadBuf::new(&mut reusable_buf[..limit]);
+    match Pin::new(r).poll_read(cx, &mut temp_buf) {
+        Poll::Ready(Ok(())) => {
+            buf.put_slice(temp_buf.filled());
+            Poll::Ready(Ok(()))
+        }
+        other => other,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures_util::task::noop_waker_ref;
+    use std::io;
+    use std::pin::Pin;
+    use std::task::Context;
+    use tokio::io::AsyncReadExt;
+
+    #[derive(Default)]
+    struct TestAsyncReader {
+        data: Vec<u8>,
+        pos: usize,
+    }
+
+    impl TestAsyncReader {
+        fn new(data: &[u8]) -> Self {
+            Self {
+                data: data.to_vec(),
+                pos: 0,
+            }
+        }
+    }
+
+    impl AsyncRead for TestAsyncReader {
+        fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
+            if self.pos >= self.data.len() {
+                return Poll::Ready(Ok(()));
+            }
+
+            let remaining = self.data.len() - self.pos;
+            let n = remaining.min(buf.remaining());
+            buf.put_slice(&self.data[self.pos..self.pos + n]);
+            self.pos += n;
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_passthrough_when_throttle_absent() {
+        let monitor = Monitor::new(1);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 0,
+        };
+        let inner = TestAsyncReader::new(b"hello-world");
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out, b"hello-world");
+    }
+
+    #[test]
+    fn test_poll_limited_read_honors_limit() {
+        let mut inner = TestAsyncReader::new(b"abcdef");
+        let mut out = [0u8; 8];
+        let mut reusable = Vec::new();
+        let waker = noop_waker_ref();
+        let mut cx = Context::from_waker(waker);
+
+        let mut read_buf = ReadBuf::new(&mut out);
+        let first = poll_limited_read(&mut inner, &mut cx, &mut read_buf, 3, &mut reusable);
+        assert!(matches!(first, Poll::Ready(Ok(()))));
+        assert_eq!(read_buf.filled(), b"abc");
+
+        let second = poll_limited_read(&mut inner, &mut cx, &mut read_buf, 3, &mut reusable);
+        assert!(matches!(second, Poll::Ready(Ok(()))));
+        assert_eq!(read_buf.filled(), b"abcdef");
+    }
+
+    #[tokio::test]
+    async fn test_header_only_consumption_skips_io() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 10);
+
+        let data = vec![0xAAu8; 20];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 25,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let waker = noop_waker_ref();
+        let mut cx = Context::from_waker(waker);
+        let mut out = [0u8; 16];
+        let mut read_buf = ReadBuf::new(&mut out);
+
+        let poll = Pin::new(&mut reader).poll_read(&mut cx, &mut read_buf);
+        assert!(matches!(poll, Poll::Ready(Ok(())) | Poll::Pending));
+        assert!(read_buf.filled().is_empty());
+        assert_eq!(reader.opts.header_size, 15);
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_with_throttle_reads_all_data() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 1024);
+
+        let data = vec![0xABu8; 4096];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 0,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out.len(), 4096);
+        assert!(out.iter().all(|&b| b == 0xAB));
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_header_size_accounting() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 100);
+
+        let data = vec![0u8; 200];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 50,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out.len(), 200);
+        assert_eq!(reader.opts.header_size, 0);
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_very_small_limit() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 1);
+
+        let data = vec![0xFFu8; 10];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 0,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out.len(), 10);
+    }
+}

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -21,6 +21,7 @@ use crate::bucket::target::ARN;
 use crate::bucket::target::BucketTargetType;
 use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials};
 use crate::bucket::versioning_sys::BucketVersioningSys;
+use crate::global::get_global_bucket_monitor;
 use aws_credential_types::Credentials as SdkCredentials;
 use aws_sdk_s3::config::Region as SdkRegion;
 use aws_sdk_s3::error::ProvideErrorMetadata;
@@ -35,8 +36,10 @@ use aws_sdk_s3::types::{
 };
 use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput};
 use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus};
+use aws_smithy_http_client::{Builder as SmithyHttpClientBuilder, tls as smithy_tls};
 use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
 use reqwest::Client as HttpClient;
+use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
 use rustfs_filemeta::{ReplicationStatusType, ReplicationType};
 use rustfs_utils::http::{
     AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
@@ -49,6 +52,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::error::Error;
 use std::fmt;
+use std::path::Path;
 use std::str::FromStr as _;
 use std::sync::Arc;
 use std::sync::OnceLock;
@@ -639,12 +643,23 @@ impl BucketTargetSys {
             format!("http://{}", target.endpoint)
         };
 
-        let config = S3Config::builder()
+        let mut config_builder = S3Config::builder()
             .endpoint_url(endpoint.clone())
             .credentials_provider(SharedCredentialsProvider::new(creds))
             .region(SdkRegion::new(target.region.clone()))
-            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
-            .build();
+            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
+
+        if should_force_path_style(target) {
+            config_builder = config_builder.force_path_style(true);
+        }
+
+        if target.secure
+            && let Some(http_client) = build_aws_s3_http_client_from_tls_path().await
+        {
+            config_builder = config_builder.http_client(http_client);
+        }
+
+        let config = config_builder.build();
 
         Ok(TargetClient {
             endpoint,
@@ -667,9 +682,19 @@ impl BucketTargetSys {
         Ok(true)
     }
 
-    fn update_bandwidth_limit(&self, _bucket: &str, _arn: &str, _limit: i64) {
-        // Implementation for bandwidth limit update
-        // This would interact with the global bucket monitor
+    fn update_bandwidth_limit(&self, bucket: &str, arn: &str, limit: i64) {
+        if let Some(bucket_monitor) = get_global_bucket_monitor() {
+            if limit == 0 {
+                bucket_monitor.delete_bucket_throttle(bucket, arn);
+                return;
+            }
+            bucket_monitor.set_bandwidth_limit(bucket, arn, limit);
+        } else {
+            error!(
+                "Global bucket monitor uninitialized; skipping bandwidth limit update for bucket '{}' and ARN '{}'",
+                bucket, arn
+            );
+        }
     }
 
     pub async fn get_remote_target_client_by_arn(&self, _bucket: &str, arn: &str) -> Option<Arc<TargetClient>> {
@@ -691,6 +716,7 @@ impl BucketTargetSys {
         if let Some(existing_targets) = targets_map.remove(bucket) {
             for target in existing_targets {
                 arn_remotes_map.remove(&target.arn);
+                self.update_bandwidth_limit(bucket, &target.arn, 0);
             }
         }
 
@@ -777,6 +803,71 @@ impl BucketTargetSys {
     }
 }
 
+async fn build_aws_s3_http_client_from_tls_path() -> Option<aws_sdk_s3::config::SharedHttpClient> {
+    let tls_path = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
+    if tls_path.is_empty() {
+        return None;
+    }
+
+    let tls_dir = Path::new(&tls_path);
+    let mut trust_store = smithy_tls::TrustStore::default();
+    let mut has_custom_certs = false;
+
+    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
+    match tokio::fs::read(&ca_path).await {
+        Ok(pem) => {
+            trust_store.add_pem_certificate(pem);
+            has_custom_certs = true;
+        }
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
+    }
+
+    if rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) {
+        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
+        match tokio::fs::read(&leaf_cert_path).await {
+            Ok(pem) => {
+                trust_store.add_pem_certificate(pem);
+                has_custom_certs = true;
+            }
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
+        }
+    }
+
+    if !has_custom_certs {
+        return None;
+    }
+
+    let tls_context = match smithy_tls::TlsContext::builder().with_trust_store(trust_store).build() {
+        Ok(ctx) => ctx,
+        Err(e) => {
+            warn!("failed to build AWS SDK TLS context for replication client: {}", e);
+            return None;
+        }
+    };
+
+    Some(
+        SmithyHttpClientBuilder::new()
+            .tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
+            .tls_context(tls_context)
+            .build_https(),
+    )
+}
+
+fn should_force_path_style(target: &BucketTarget) -> bool {
+    match target.path.trim().to_ascii_lowercase().as_str() {
+        // Explicit DNS/virtual-hosted-style requested by user.
+        "dns" | "off" | "false" => false,
+        // Explicit path-style or legacy boolean-like values.
+        "path" | "on" | "true" => true,
+        // `auto` and empty are defaulted to path-style for custom S3-compatible endpoints.
+        "auto" | "" => true,
+        // Unknown values: prefer compatibility with S3-compatible services.
+        _ => true,
+    }
+}
+
 // generate ARN that is unique to this target type
 fn generate_arn(t: &BucketTarget, depl_id: &str) -> String {
     let uuid = if depl_id.is_empty() {

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -28,13 +28,12 @@ use crate::client::object_api_utils::new_getobjectreader;
 use crate::error::Error;
 use crate::error::StorageError;
 use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down};
-use crate::event::name::EventName;
 use crate::event_notification::{EventArgs, send_event};
 use crate::global::GLOBAL_LocalNodeName;
 use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
 use crate::store::ECStore;
 use crate::store_api::StorageAPI;
-use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
+use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete};
 use crate::tier::warm_backend::WarmBackendGetOpts;
 use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
 use bytes::BytesMut;
@@ -45,6 +44,7 @@ use rustfs_common::data_usage::TierStats;
 use rustfs_common::heal_channel::rep_has_active_rules;
 use rustfs_common::metrics::{IlmAction, Metrics};
 use rustfs_filemeta::{NULL_VERSION_ID, RestoreStatusOps, is_restored_object_on_disk};
+use rustfs_s3_common::EventName;
 use rustfs_utils::path::encode_dir_object;
 use rustfs_utils::string::strings_has_prefix_fold;
 use s3s::Body;
@@ -65,7 +65,7 @@ use time::OffsetDateTime;
 use tokio::select;
 use tokio::sync::mpsc::{Receiver, Sender};
 use tokio::sync::{RwLock, mpsc};
-use tracing::{debug, error, info};
+use tracing::{debug, error, info, warn};
 use uuid::Uuid;
 use xxhash_rust::xxh64;
 
@@ -102,7 +102,7 @@ impl LifecycleSys {
     }
 
     pub fn trace(_oi: &ObjectInfo) -> TraceFn {
-        todo!();
+        Arc::new(|_oi, _ctx| Box::pin(async move {}))
     }
 }
 
@@ -395,7 +395,7 @@ impl ExpiryState {
                     }
                     else {
                         //info!("Invalid work type - {:?}", v);
-                        todo!();
+                        warn!("lifecycle worker received unsupported operation type");
                     }
                 }
             }
@@ -471,7 +471,7 @@ impl TransitionState {
     }
 
     pub async fn init(api: Arc<ECStore>) {
-        let max_workers = std::env::var("RUSTFS_MAX_TRANSITION_WORKERS")
+        let max_workers = env::var("RUSTFS_MAX_TRANSITION_WORKERS")
             .ok()
             .and_then(|s| s.parse::<i64>().ok())
             .unwrap_or_else(|| std::cmp::min(num_cpus::get() as i64, 16));
@@ -569,14 +569,14 @@ impl TransitionState {
     pub async fn update_workers_inner(api: Arc<ECStore>, n: i64) {
         let mut n = n;
         if n == 0 {
-            let max_workers = std::env::var("RUSTFS_MAX_TRANSITION_WORKERS")
+            let max_workers = env::var("RUSTFS_MAX_TRANSITION_WORKERS")
                 .ok()
                 .and_then(|s| s.parse::<i64>().ok())
                 .unwrap_or_else(|| std::cmp::min(num_cpus::get() as i64, 16));
             n = max_workers;
         }
         // Allow environment override of maximum workers
-        let absolute_max = std::env::var("RUSTFS_ABSOLUTE_MAX_WORKERS")
+        let absolute_max = env::var("RUSTFS_ABSOLUTE_MAX_WORKERS")
             .ok()
             .and_then(|s| s.parse::<i64>().ok())
             .unwrap_or(32);
@@ -603,7 +603,7 @@ impl TransitionState {
 }
 
 pub async fn init_background_expiry(api: Arc<ECStore>) {
-    let mut workers = std::env::var("RUSTFS_MAX_EXPIRY_WORKERS")
+    let mut workers = env::var("RUSTFS_MAX_EXPIRY_WORKERS")
         .ok()
         .and_then(|s| s.parse::<usize>().ok())
         .unwrap_or_else(|| std::cmp::min(num_cpus::get(), 16));
@@ -615,7 +615,7 @@ pub async fn init_background_expiry(api: Arc<ECStore>) {
     }
 
     if workers == 0 {
-        workers = std::env::var("RUSTFS_DEFAULT_EXPIRY_WORKERS")
+        workers = env::var("RUSTFS_DEFAULT_EXPIRY_WORKERS")
             .ok()
             .and_then(|s| s.parse::<usize>().ok())
             .unwrap_or(8);
@@ -689,13 +689,13 @@ pub async fn expire_transitioned_object(
     //let tags = LcAuditEvent::new(src, lcEvent).Tags();
     if lc_event.action == IlmAction::DeleteRestoredAction {
         opts.transition.expire_restored = true;
-        match api.delete_object(&oi.bucket, &oi.name, opts).await {
+        return match api.delete_object(&oi.bucket, &oi.name, opts).await {
             Ok(dobj) => {
                 //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn);
-                return Ok(dobj);
+                Ok(dobj)
             }
-            Err(err) => return Err(std::io::Error::other(err)),
-        }
+            Err(err) => Err(std::io::Error::other(err)),
+        };
     }
 
     let ret = delete_object_from_remote_tier(
@@ -732,7 +732,7 @@ pub async fn expire_transitioned_object(
         ..Default::default()
     };
     send_event(EventArgs {
-        event_name: event_name.as_ref().to_string(),
+        event_name: event_name.to_string(),
         bucket_name: obj_info.bucket.clone(),
         object: obj_info,
         user_agent: "Internal: [ILM-Expiry]".to_string(),
@@ -782,12 +782,13 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
         mod_time: oi.mod_time,
         ..Default::default()
     };
-    time_ilm(1);
-    api.transition_object(&oi.bucket, &oi.name, &opts).await
+    let result = api.transition_object(&oi.bucket, &oi.name, &opts).await;
+    time_ilm(1)();
+    result
 }
 
 pub fn audit_tier_actions(_api: ECStore, _tier: &str, _bytes: i64) -> TimeFn {
-    todo!();
+    Arc::new(|| Box::pin(async move {}))
 }
 
 pub async fn get_transitioned_object_reader(
@@ -846,8 +847,8 @@ pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) ->
         }
     }
     Ok(ObjectOptions {
-        versioned: versioned,
-        version_suspended: version_suspended,
+        versioned,
+        version_suspended,
         version_id: Some(vid.to_string()),
         ..Default::default()
     })
@@ -1032,12 +1033,12 @@ pub async fn eval_action_from_lifecycle(
     let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false };
 
     match event.action {
-        lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
+        IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
             if lock_enabled {
                 return lifecycle::Event::default();
             }
         }
-        lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteRestoredVersionAction => {
+        IlmAction::DeleteVersionAction | IlmAction::DeleteRestoredVersionAction => {
             if oi.version_id.is_none() {
                 return lifecycle::Event::default();
             }
@@ -1082,11 +1083,11 @@ pub async fn apply_expiry_on_transitioned_object(
     lc_event: &lifecycle::Event,
     src: &LcEventSrc,
 ) -> bool {
-    // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
+    let time_ilm = Metrics::time_ilm(lc_event.action);
     if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await {
         return false;
     }
-    // let _ = time_ilm(1);
+    time_ilm(1)();
 
     true
 }
@@ -1114,7 +1115,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
         opts.delete_prefix_object = true;
     }
 
-    // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
+    let time_ilm = Metrics::time_ilm(lc_event.action);
 
     //debug!("lc_event.action: {:?}", lc_event.action);
     //debug!("opts: {:?}", opts);
@@ -1138,12 +1139,12 @@ pub async fn apply_expiry_on_non_transitioned_objects(
         event_name = EventName::ObjectRemovedDeleteMarkerCreated;
     }
     match lc_event.action {
-        lifecycle::IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions,
-        lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::ILMDelMarkerExpirationDelete,
+        IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions,
+        IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::LifecycleDelMarkerExpirationDelete,
         _ => (),
     }
     send_event(EventArgs {
-        event_name: event_name.as_ref().to_string(),
+        event_name: event_name.to_string(),
         bucket_name: dobj.bucket.clone(),
         object: dobj,
         user_agent: "Internal: [ILM-Expiry]".to_string(),
@@ -1151,12 +1152,12 @@ pub async fn apply_expiry_on_non_transitioned_objects(
         ..Default::default()
     });
 
-    if lc_event.action != lifecycle::IlmAction::NoneAction {
-        // let mut num_versions = 1_u64;
-        // if lc_event.action.delete_all() {
-        //     num_versions = oi.num_versions as u64;
-        // }
-        // let _ = time_ilm(num_versions);
+    if lc_event.action != IlmAction::NoneAction {
+        let mut num_versions = 1_u64;
+        if lc_event.action.delete_all() {
+            num_versions = oi.num_versions as u64;
+        }
+        time_ilm(num_versions)();
     }
 
     true
@@ -1171,15 +1172,15 @@ pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &
 pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
     let mut success = false;
     match event.action {
-        lifecycle::IlmAction::DeleteVersionAction
-        | lifecycle::IlmAction::DeleteAction
-        | lifecycle::IlmAction::DeleteRestoredAction
-        | lifecycle::IlmAction::DeleteRestoredVersionAction
-        | lifecycle::IlmAction::DeleteAllVersionsAction
-        | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
+        IlmAction::DeleteVersionAction
+        | IlmAction::DeleteAction
+        | IlmAction::DeleteRestoredAction
+        | IlmAction::DeleteRestoredVersionAction
+        | IlmAction::DeleteAllVersionsAction
+        | IlmAction::DelMarkerDeleteAllVersionsAction => {
             success = apply_expiry_rule(event, src, oi).await;
         }
-        lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
+        IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
             success = apply_transition_rule(event, src, oi).await;
         }
         _ => (),

crates/ecstore/src/bucket/lifecycle/lifecycle.rs

@@ -45,6 +45,10 @@ const _ERR_XML_NOT_WELL_FORMED: &str =
 const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
     "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an retention bucket";
 const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
+const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days must be greater than 0";
+const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
+const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters";
+const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
 
 pub use rustfs_common::metrics::IlmAction;
 
@@ -138,6 +142,7 @@ pub trait Lifecycle {
     async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error>;
     async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>>;
     async fn eval(&self, obj: &ObjectOpts) -> Event;
+    async fn predict_expiration(&self, obj: &ObjectOpts) -> Event;
     async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime, newer_noncurrent_versions: usize) -> Event;
     //fn set_prediction_headers(&self, w: http.ResponseWriter, obj: ObjectOpts);
     async fn noncurrent_versions_expiration_limit(self: Arc<Self>, obj: &ObjectOpts) -> Event;
@@ -232,6 +237,30 @@ impl Lifecycle for BucketLifecycleConfiguration {
         }
 
         for r in &self.rules {
+            if r.status != ExpirationStatus::from_static(ExpirationStatus::ENABLED)
+                && r.status != ExpirationStatus::from_static(ExpirationStatus::DISABLED)
+            {
+                return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_STATUS));
+            }
+
+            if let Some(expiration) = &r.expiration {
+                if let Some(expiration_date) = &expiration.date {
+                    let date = OffsetDateTime::from(expiration_date.clone());
+                    if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT));
+                    }
+                }
+                if let Some(days) = expiration.days {
+                    if days <= 0 {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS));
+                    }
+                }
+            }
+            if let Some(id) = &r.id {
+                if id.len() > 255 {
+                    return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG));
+                }
+            }
             r.validate()?;
             /*if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() {
                 if let Some(expiration) = r.expiration.as_ref() {
@@ -249,8 +278,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
             }
             let other_rules = &self.rules[i + 1..];
             for other_rule in other_rules {
-                if self.rules[i].id == other_rule.id {
-                    return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
+                if let (Some(id1), Some(id2)) = (&self.rules[i].id, &other_rule.id) {
+                    if id1 == id2 {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
+                    }
                 }
             }
         }
@@ -287,6 +318,61 @@ impl Lifecycle for BucketLifecycleConfiguration {
         self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await
     }
 
+    async fn predict_expiration(&self, obj: &ObjectOpts) -> Event {
+        let mod_time = match obj.mod_time {
+            Some(time) => {
+                if time.unix_timestamp() == 0 {
+                    return Event::default();
+                }
+                time
+            }
+            None => return Event::default(),
+        };
+
+        if obj.delete_marker || !(obj.is_latest || obj.version_id.is_none_or(|v| v.is_nil())) {
+            return Event::default();
+        }
+
+        let Some(lc_rules) = self.filter_rules(obj).await else {
+            return Event::default();
+        };
+
+        let mut event: Option<Event> = None;
+        for rule in lc_rules {
+            let Some(expiration) = rule.expiration else {
+                continue;
+            };
+            if expiration.expired_object_delete_marker.is_some_and(|v| v) {
+                continue;
+            }
+            let Some(due) = (match (&expiration.date, expiration.days) {
+                (Some(date), _) => Some(OffsetDateTime::from(date.clone())),
+                (None, Some(days)) => Some(expected_expiry_time(mod_time, days)),
+                _ => None,
+            }) else {
+                continue;
+            };
+
+            let predicted = Event {
+                action: IlmAction::DeleteAction,
+                rule_id: rule.id.clone().unwrap_or_default(),
+                due: Some(due),
+                noncurrent_days: 0,
+                newer_noncurrent_versions: 0,
+                storage_class: "".into(),
+            };
+            let predicted_due = predicted.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp();
+            let should_replace = event
+                .as_ref()
+                .is_none_or(|e| predicted_due < e.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp());
+            if should_replace {
+                event = Some(predicted);
+            }
+        }
+
+        event.unwrap_or_default()
+    }
+
     async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime, newer_noncurrent_versions: usize) -> Event {
         let mut events = Vec::<Event>::new();
         info!(
@@ -770,3 +856,222 @@ impl Default for TransitionOptions {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn validate_rejects_non_positive_expiration_days() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(0),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc
+            .validate(&ObjectLockConfiguration::default())
+            .await
+            .expect_err("expected validation error");
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS);
+    }
+
+    #[tokio::test]
+    async fn validate_accepts_positive_expiration_days() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        lc.validate(&ObjectLockConfiguration::default())
+            .await
+            .expect("expected validation to pass");
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_non_midnight_expiration_date() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    date: Some(time::OffsetDateTime::from_unix_timestamp(20_000_101).unwrap().into()),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT);
+    }
+
+    #[tokio::test]
+    async fn predict_expiration_selects_closest_expiry_for_put_object() {
+        let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        days: Some(30),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: Some("rule-days".to_string()),
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        date: Some((base_time + Duration::days(1)).into()),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: Some("rule-date".to_string()),
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+            ],
+        };
+        let opts = ObjectOpts {
+            name: "obj".to_string(),
+            mod_time: Some(base_time),
+            is_latest: true,
+            version_id: None,
+            ..Default::default()
+        };
+
+        let event = lc.predict_expiration(&opts).await;
+        let expected = base_time + Duration::days(1);
+
+        assert_eq!(event.action, IlmAction::DeleteAction);
+        assert_eq!(event.rule_id, "rule-date");
+        assert_eq!(event.due, Some(expected));
+    }
+
+    #[tokio::test]
+    async fn validate_accepts_multiple_rules_without_ids() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        days: Some(30),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: None,
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        days: Some(31),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: None,
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+            ],
+        };
+
+        lc.validate(&ObjectLockConfiguration::default())
+            .await
+            .expect("expected validation to pass");
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_rule_id_too_long() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: Some("a".repeat(256)),
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG);
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_invalid_status_case_sensitive() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static("enabled"),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_RULE_STATUS);
+    }
+}

crates/ecstore/src/bucket/metadata.rs

@@ -26,7 +26,8 @@ use rmp_serde::Serializer as rmpSerializer;
 use rustfs_policy::policy::BucketPolicy;
 use s3s::dto::{
     BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration,
-    ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
+    PublicAccessBlockConfiguration, ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging,
+    VersioningConfiguration,
 };
 use serde::Serializer;
 use serde::{Deserialize, Serialize};
@@ -50,6 +51,8 @@ pub const BUCKET_VERSIONING_CONFIG: &str = "versioning.xml";
 pub const BUCKET_REPLICATION_CONFIG: &str = "replication.xml";
 pub const BUCKET_TARGETS_FILE: &str = "bucket-targets.json";
 pub const BUCKET_CORS_CONFIG: &str = "cors.xml";
+pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
+pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
 
 #[derive(Debug, Deserialize, Serialize, Clone)]
 #[serde(rename_all = "PascalCase", default)]
@@ -69,6 +72,8 @@ pub struct BucketMetadata {
     pub bucket_targets_config_json: Vec<u8>,
     pub bucket_targets_config_meta_json: Vec<u8>,
     pub cors_config_xml: Vec<u8>,
+    pub public_access_block_config_xml: Vec<u8>,
+    pub bucket_acl_config_json: Vec<u8>,
 
     pub policy_config_updated_at: OffsetDateTime,
     pub object_lock_config_updated_at: OffsetDateTime,
@@ -82,6 +87,8 @@ pub struct BucketMetadata {
     pub bucket_targets_config_updated_at: OffsetDateTime,
     pub bucket_targets_config_meta_updated_at: OffsetDateTime,
     pub cors_config_updated_at: OffsetDateTime,
+    pub public_access_block_config_updated_at: OffsetDateTime,
+    pub bucket_acl_config_updated_at: OffsetDateTime,
 
     #[serde(skip)]
     pub new_field_updated_at: OffsetDateTime,
@@ -110,6 +117,10 @@ pub struct BucketMetadata {
     pub bucket_target_config_meta: Option<HashMap<String, String>>,
     #[serde(skip)]
     pub cors_config: Option<CORSConfiguration>,
+    #[serde(skip)]
+    pub public_access_block_config: Option<PublicAccessBlockConfiguration>,
+    #[serde(skip)]
+    pub bucket_acl_config: Option<String>,
 }
 
 impl Default for BucketMetadata {
@@ -130,6 +141,8 @@ impl Default for BucketMetadata {
             bucket_targets_config_json: Default::default(),
             bucket_targets_config_meta_json: Default::default(),
             cors_config_xml: Default::default(),
+            public_access_block_config_xml: Default::default(),
+            bucket_acl_config_json: Default::default(),
             policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
@@ -142,6 +155,8 @@ impl Default for BucketMetadata {
             bucket_targets_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             bucket_targets_config_meta_updated_at: OffsetDateTime::UNIX_EPOCH,
             cors_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
             policy_config: Default::default(),
             notification_config: Default::default(),
@@ -155,6 +170,8 @@ impl Default for BucketMetadata {
             bucket_target_config: Default::default(),
             bucket_target_config_meta: Default::default(),
             cors_config: Default::default(),
+            public_access_block_config: Default::default(),
+            bucket_acl_config: Default::default(),
         }
     }
 }
@@ -254,6 +271,12 @@ impl BucketMetadata {
         if self.bucket_targets_config_meta_updated_at == OffsetDateTime::UNIX_EPOCH {
             self.bucket_targets_config_meta_updated_at = self.created
         }
+        if self.public_access_block_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.public_access_block_config_updated_at = self.created
+        }
+        if self.bucket_acl_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.bucket_acl_config_updated_at = self.created
+        }
     }
 
     pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
@@ -307,6 +330,14 @@ impl BucketMetadata {
                 self.cors_config_xml = data;
                 self.cors_config_updated_at = updated;
             }
+            BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => {
+                self.public_access_block_config_xml = data;
+                self.public_access_block_config_updated_at = updated;
+            }
+            BUCKET_ACL_CONFIG => {
+                self.bucket_acl_config_json = data;
+                self.bucket_acl_config_updated_at = updated;
+            }
             _ => return Err(Error::other(format!("config file not found : {config_file}"))),
         }
 
@@ -341,10 +372,17 @@ impl BucketMetadata {
         Ok(())
     }
 
-    fn parse_all_configs(&mut self, _api: Arc<ECStore>) -> Result<()> {
+    fn parse_policy_config(&mut self) -> Result<()> {
         if !self.policy_config_json.is_empty() {
             self.policy_config = Some(serde_json::from_slice(&self.policy_config_json)?);
+        } else {
+            self.policy_config = None;
         }
+        Ok(())
+    }
+
+    fn parse_all_configs(&mut self, _api: Arc<ECStore>) -> Result<()> {
+        self.parse_policy_config()?;
         if !self.notification_config_xml.is_empty() {
             self.notification_config = Some(deserialize::<NotificationConfiguration>(&self.notification_config_xml)?);
         }
@@ -380,6 +418,15 @@ impl BucketMetadata {
         if !self.cors_config_xml.is_empty() {
             self.cors_config = Some(deserialize::<CORSConfiguration>(&self.cors_config_xml)?);
         }
+        if !self.public_access_block_config_xml.is_empty() {
+            self.public_access_block_config =
+                Some(deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)?);
+        }
+        if !self.bucket_acl_config_json.is_empty() {
+            let acl = String::from_utf8(self.bucket_acl_config_json.clone())
+                .map_err(|e| Error::other(format!("invalid UTF-8 in bucket ACL: {}", e)))?;
+            self.bucket_acl_config = Some(acl);
+        }
 
         Ok(())
     }
@@ -530,6 +577,15 @@ mod test {
         bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec();
         bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc();
 
+        // Add public access block configuration
+        let public_access_block_xml = r#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#;
+        bm.public_access_block_config_xml = public_access_block_xml.as_bytes().to_vec();
+        bm.public_access_block_config_updated_at = OffsetDateTime::now_utc();
+
+        let bucket_acl = r#"{"owner":{"id":"rustfsadmin","display_name":"RustFS Tester"},"grants":[{"grantee":{"grantee_type":"CanonicalUser","id":"rustfsadmin","display_name":"RustFS Tester","uri":null,"email_address":null},"permission":"FULL_CONTROL"}]}"#;
+        bm.bucket_acl_config_json = bucket_acl.as_bytes().to_vec();
+        bm.bucket_acl_config_updated_at = OffsetDateTime::now_utc();
+
         // Test serialization
         let buf = bm.marshal_msg().unwrap();
         assert!(!buf.is_empty(), "Serialized buffer should not be empty");
@@ -549,6 +605,8 @@ mod test {
         assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml);
         assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml);
         assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json);
+        assert_eq!(bm.public_access_block_config_xml, deserialized_bm.public_access_block_config_xml);
+        assert_eq!(bm.bucket_acl_config_json, deserialized_bm.bucket_acl_config_json);
         assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml);
         assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml);
         assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml);
@@ -615,4 +673,21 @@ mod test {
         println!("   - Lifecycle config size: {} bytes", deserialized_bm.lifecycle_config_xml.len());
         println!("   - Serialized buffer size: {} bytes", buf.len());
     }
+
+    /// After policy deletion (policy_config_json cleared), parse_policy_config sets policy_config to None.
+    #[test]
+    fn test_parse_policy_config_clears_cache_when_json_empty() {
+        let policy_json = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}"#;
+        let mut bm = BucketMetadata::new("b");
+        bm.policy_config_json = policy_json.as_bytes().to_vec();
+        bm.parse_policy_config().unwrap();
+        assert!(bm.policy_config.is_some(), "policy_config should be set when JSON non-empty");
+
+        bm.policy_config_json.clear();
+        bm.parse_policy_config().unwrap();
+        assert!(
+            bm.policy_config.is_none(),
+            "policy_config should be None after JSON cleared (e.g. policy deleted)"
+        );
+    }
 }

crates/ecstore/src/bucket/metadata_sys.rs

@@ -15,13 +15,13 @@
 use super::metadata::{BucketMetadata, load_bucket_metadata};
 use super::quota::BucketQuota;
 use super::target::BucketTargets;
-use crate::StorageAPI as _;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_parse};
 use crate::bucket::utils::{deserialize, is_meta_bucketname};
 use crate::error::{Error, Result, is_err_bucket_not_found};
 use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_layer_fn};
 use crate::store::ECStore;
+use crate::store_api::HealOperations as _;
 use futures::future::join_all;
 use lazy_static::lazy_static;
 use rustfs_common::heal_channel::HealOpts;
@@ -29,7 +29,7 @@ use rustfs_policy::policy::BucketPolicy;
 use s3s::dto::ReplicationConfiguration;
 use s3s::dto::{
     BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration,
-    ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
+    PublicAccessBlockConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
 };
 use std::collections::HashSet;
 use std::sync::OnceLock;
@@ -53,6 +53,10 @@ pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
     GLOBAL_BucketMetadataSys.set(sys).unwrap();
 }
 
+pub fn get_global_bucket_metadata_sys() -> Option<Arc<RwLock<BucketMetadataSys>>> {
+    GLOBAL_BucketMetadataSys.get().cloned()
+}
+
 // panic if not init
 pub(super) fn get_bucket_metadata_sys() -> Result<Arc<RwLock<BucketMetadataSys>>> {
     if let Some(sys) = GLOBAL_BucketMetadataSys.get() {
@@ -105,6 +109,13 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
     bucket_meta_sys.get_bucket_policy_raw(bucket).await
 }
 
+pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_bucket_acl_config(bucket).await
+}
+
 pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -133,6 +144,13 @@ pub async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime
     bucket_meta_sys.get_tagging_config(bucket).await
 }
 
+pub async fn get_public_access_block_config(bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_public_access_block_config(bucket).await
+}
+
 pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -478,6 +496,16 @@ impl BucketMetadataSys {
         }
     }
 
+    pub async fn get_bucket_acl_config(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.bucket_acl_config {
+            Ok((config.clone(), bm.bucket_acl_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
     pub async fn get_tagging_config(&self, bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
@@ -488,6 +516,16 @@ impl BucketMetadataSys {
         }
     }
 
+    pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.public_access_block_config {
+            Ok((config.clone(), bm.public_access_block_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
     pub async fn get_object_lock_config(&self, bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 

crates/ecstore/src/bucket/mod.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod bandwidth;
 pub mod bucket_target_sys;
 pub mod error;
 pub mod lifecycle;