fix: restore default CORS fallback and STS object ACL ownership (#2053)

Co-authored-by: houseme <housemecn@gmail.com>

.config/make/tests.mak

@@ -1,5 +1,7 @@
 ## —— Tests and e2e test ---------------------------------------------------------------------------
 
+TEST_THREADS ?= 1
+
 .PHONY: test
 test: core-deps test-deps ## Run all tests
 	@echo "🧪 Running tests..."
@@ -7,7 +9,7 @@ test: core-deps test-deps ## Run all tests
 		cargo nextest run --all --exclude e2e_test; \
 	else \
 		echo "ℹ️ cargo-nextest not found; falling back to 'cargo test'"; \
-		cargo test --workspace --exclude e2e_test -- --nocapture; \
+		cargo test --workspace --exclude e2e_test -- --nocapture --test-threads="$(TEST_THREADS)"; \
 	fi
 	cargo test --all --doc
 
@@ -17,4 +19,4 @@ e2e-server: ## Run e2e-server tests
 
 .PHONY: probe-e2e
 probe-e2e: ## Probe e2e tests
-	sh $(shell pwd)/scripts/probe.sh
+	sh $(shell pwd)/scripts/probe.sh

.docker/README.md

@@ -1,261 +1,131 @@
-# RustFS Docker Images
+# RustFS Docker Infrastructure
 
-This directory contains Docker configuration files and supporting infrastructure for building and running RustFS container images.
+This directory contains the complete Docker infrastructure for building, deploying, and monitoring RustFS. It provides ready-to-use configurations for development, testing, and production-grade observability.
 
-## 📁 Directory Structure
+## 📂 Directory Structure
 
-```
-rustfs/
-├── Dockerfile           # Production image (Alpine + pre-built binaries)
-├── Dockerfile.source    # Development image (Debian + source build)
-├── docker-buildx.sh     # Multi-architecture build script
-├── Makefile             # Build automation with simplified commands
-└── .docker/             # Supporting infrastructure
-    ├── observability/   # Monitoring and observability configs
-    ├── compose/         # Docker Compose configurations
-    ├── mqtt/            # MQTT broker configs
-    └── openobserve-otel/ # OpenObserve + OpenTelemetry configs
-```
+| Directory | Description | Status |
+| :--- | :--- | :--- |
+| **[`observability/`](observability/README.md)** | **[RECOMMENDED]** Full-stack observability (Prometheus, Grafana, Tempo, Loki). | ✅ Production-Ready |
+| **[`compose/`](compose/README.md)** | Specialized setups (e.g., 4-node distributed cluster testing). | ⚠️ Testing Only |
+| **[`mqtt/`](mqtt/README.md)** | EMQX Broker configuration for MQTT integration testing. | 🧪 Development |
+| **[`openobserve-otel/`](openobserve-otel/README.md)** | Alternative lightweight observability stack using OpenObserve. | 🔄 Alternative |
 
-## 🎯 Image Variants
+---
 
-### Core Images
+## 📄 Root Directory Files
 
-| Image | Base OS | Build Method | Size | Use Case |
-|-------|---------|--------------|------|----------|
-| `production` (default) | Alpine 3.18 | GitHub Releases | Smallest | Production deployment |
-| `source` | Debian Bookworm | Source build | Medium | Custom builds with cross-compilation |
-| `dev` | Debian Bookworm | Development tools | Large | Interactive development |
+The following files in the project root are essential for Docker operations:
 
-## 🚀 Usage Examples
+### Build Scripts & Dockerfiles
 
-### Quick Start (Production)
+| File | Description | Usage |
+| :--- | :--- | :--- |
+| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
+| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
+| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
+| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
 
+### Docker Compose Configurations
+
+| File | Description | Usage |
+| :--- | :--- | :--- |
+| **`docker-compose.yml`** | **Main Development Setup**<br>Comprehensive setup with profiles for development, observability, and proxying. | `docker compose up -d`<br>`docker compose --profile observability up -d` |
+| **`docker-compose-simple.yml`** | **Quick Start Setup**<br>Minimal configuration running a single RustFS instance with 4 volumes. Perfect for first-time users. | `docker compose -f docker-compose-simple.yml up -d` |
+
+---
+
+## 🌟 Observability Stack (Recommended)
+
+Located in: [`.docker/observability/`](observability/README.md)
+
+We provide a comprehensive, industry-standard observability stack designed for deep insights into RustFS performance. This is the recommended setup for both development and production monitoring.
+
+### Components
+- **Metrics**: Prometheus (Collection) + Grafana (Visualization)
+- **Traces**: Tempo (Storage) + Jaeger (UI)
+- **Logs**: Loki
+- **Ingestion**: OpenTelemetry Collector
+
+### Key Features
+- **Full Persistence**: All metrics, logs, and traces are saved to Docker volumes, ensuring no data loss on restarts.
+- **Correlation**: Seamlessly jump between Logs, Traces, and Metrics in Grafana.
+- **High Performance**: Optimized configurations for batching, compression, and memory management.
+
+### Quick Start
 ```bash
-# Default production image (Alpine + GitHub Releases)
-docker run -p 9000:9000 rustfs/rustfs:latest
-
-# Specific version
-docker run -p 9000:9000 rustfs/rustfs:1.2.3
+cd .docker/observability
+docker compose up -d
 ```
 
-### Complete Tag Strategy Examples
+---
 
+## 🧪 Specialized Environments
+
+Located in: [`.docker/compose/`](compose/README.md)
+
+These configurations are tailored for specific testing scenarios that require complex topologies.
+
+### Distributed Cluster (4-Nodes)
+Simulates a real-world distributed environment with 4 RustFS nodes running locally.
 ```bash
-# Stable Releases
-docker run rustfs/rustfs:1.2.3           # Main version (production)
-docker run rustfs/rustfs:1.2.3-production # Explicit production variant
-docker run rustfs/rustfs:1.2.3-source   # Source build variant
-docker run rustfs/rustfs:latest          # Latest stable
-
-# Prerelease Versions
-docker run rustfs/rustfs:1.3.0-alpha.2  # Specific alpha version
-docker run rustfs/rustfs:alpha           # Latest alpha
-docker run rustfs/rustfs:beta            # Latest beta
-docker run rustfs/rustfs:rc              # Latest release candidate
-
-# Development Versions
-docker run rustfs/rustfs:dev             # Latest main branch development
-docker run rustfs/rustfs:dev-13e4a0b     # Specific commit
-docker run rustfs/rustfs:dev-latest      # Latest development
-docker run rustfs/rustfs:main-latest     # Main branch latest
+docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
 ```
 
-### Development Environment
-
+### Integrated Observability Test
+A self-contained environment running 4 RustFS nodes alongside the full observability stack. Useful for end-to-end testing of telemetry.
 ```bash
-# Quick setup using Makefile (recommended)
-make docker-dev-local     # Build development image locally
-make dev-env-start        # Start development container
-
-# Manual Docker commands
-docker run -it -v $(pwd):/workspace -p 9000:9000 rustfs/rustfs:latest-dev
-
-# Build from source locally
-docker build -f Dockerfile.source -t rustfs:custom .
-
-# Development with hot reload
-docker-compose up rustfs-dev
+docker compose -f .docker/compose/docker-compose.observability.yaml up -d
 ```
 
-## 🏗️ Build Arguments and Scripts
+---
 
-### Using Makefile Commands (Recommended)
+## 📡 MQTT Integration
 
-The easiest way to build images using simplified commands:
+Located in: [`.docker/mqtt/`](mqtt/README.md)
 
+Provides an EMQX broker for testing RustFS MQTT features.
+
+### Quick Start
 ```bash
-# Development images (build from source)
-make docker-dev-local             # Build for local use (single arch)
-make docker-dev                   # Build multi-arch (for CI/CD)
-make docker-dev-push REGISTRY=xxx # Build and push to registry
-
-# Production images (using pre-built binaries)
-make docker-buildx                # Build multi-arch production images
-make docker-buildx-push           # Build and push production images
-make docker-buildx-version VERSION=v1.0.0  # Build specific version
-
-# Development environment
-make dev-env-start                # Start development container
-make dev-env-stop                 # Stop development container
-make dev-env-restart              # Restart development container
-
-# Help
-make help-docker                  # Show all Docker-related commands
+cd .docker/mqtt
+docker compose up -d
 ```
+- **Dashboard**: [http://localhost:18083](http://localhost:18083) (Default: `admin` / `public`)
+- **MQTT Port**: `1883`
 
-### Using docker-buildx.sh (Advanced)
+---
 
-For direct script usage and advanced scenarios:
+## 👁️ Alternative: OpenObserve
 
+Located in: [`.docker/openobserve-otel/`](openobserve-otel/README.md)
+
+For users preferring a lightweight, all-in-one solution, we support OpenObserve. It combines logs, metrics, and traces into a single binary and UI.
+
+### Quick Start
 ```bash
-# Build latest version for all architectures
-./docker-buildx.sh
-
-# Build and push to registry
-./docker-buildx.sh --push
-
-# Build specific version
-./docker-buildx.sh --release v1.2.3
-
-# Build and push specific version
-./docker-buildx.sh --release v1.2.3 --push
+cd .docker/openobserve-otel
+docker compose up -d
 ```
 
-### Manual Docker Builds
+---
 
-All images support dynamic version selection:
+## 🔧 Common Operations
 
+### Cleaning Up
+To stop all containers and remove volumes (**WARNING**: deletes all persisted data):
 ```bash
-# Build production image with latest release
-docker build --build-arg RELEASE="latest" -t rustfs:latest .
-
-# Build from source with specific target
-docker build -f Dockerfile.source \
-  --build-arg TARGETPLATFORM="linux/amd64" \
-  -t rustfs:source .
-
-# Development build
-docker build -f Dockerfile.source -t rustfs:dev .
+docker compose down -v
 ```
 
-## 🔧 Binary Download Sources
-
-### Unified GitHub Releases
-
-The production image downloads from GitHub Releases for reliability and transparency:
-
-- ✅ **production** → GitHub Releases API with automatic latest detection
-- ✅ **Checksum verification** → SHA256SUMS validation when available
-- ✅ **Multi-architecture** → Supports amd64 and arm64
-
-### Source Build
-
-The source variant compiles from source code with advanced features:
-
-- 🔧 **Cross-compilation** → Supports multiple target platforms via `TARGETPLATFORM`
-- ⚡ **Build caching** → sccache for faster compilation
-- 🎯 **Optimized builds** → Release optimizations with LTO and symbol stripping
-
-## 📋 Architecture Support
-
-All variants support multi-architecture builds:
-
-- **linux/amd64** (x86_64)
-- **linux/arm64** (aarch64)
-
-Architecture is automatically detected during build using Docker's `TARGETARCH` build argument.
-
-## 🔐 Security Features
-
-- **Checksum Verification**: Production image verifies SHA256SUMS when available
-- **Non-root User**: All images run as user `rustfs` (UID 1000)
-- **Minimal Runtime**: Production image only includes necessary dependencies
-- **Secure Defaults**: No hardcoded credentials or keys
-
-## 🛠️ Development Workflow
-
-### Quick Start with Makefile (Recommended)
-
+### Viewing Logs
+To follow logs for a specific service:
 ```bash
-# 1. Start development environment
-make dev-env-start
-
-# 2. Your development container is now running with:
-#    - Port 9000 exposed for RustFS
-#    - Port 9010 exposed for admin console
-#    - Current directory mounted as /workspace
-
-# 3. Stop when done
-make dev-env-stop
+docker compose logs -f [service_name]
 ```
 
-### Manual Development Setup
-
+### Checking Status
+To see the status of all running containers:
 ```bash
-# Build development image from source
-make docker-dev-local
-
-# Or use traditional Docker commands
-docker build -f Dockerfile.source -t rustfs:dev .
-
-# Run with development tools
-docker run -it -v $(pwd):/workspace -p 9000:9000 rustfs:dev bash
-
-# Or use docker-compose for complex setups
-docker-compose up rustfs-dev
+docker compose ps
 ```
-
-### Common Development Tasks
-
-```bash
-# Build and test locally
-make build                        # Build binary natively
-make docker-dev-local            # Build development Docker image
-make test                        # Run tests
-make fmt                         # Format code
-make clippy                      # Run linter
-
-# Get help
-make help                        # General help
-make help-docker                 # Docker-specific help
-make help-build                  # Build-specific help
-```
-
-## 🚀 CI/CD Integration
-
-The project uses GitHub Actions for automated multi-architecture Docker builds:
-
-### Automated Builds
-
-- **Tags**: Automatic builds triggered on version tags (e.g., `v1.2.3`)
-- **Main Branch**: Development builds with `dev-latest` and `main-latest` tags
-- **Pull Requests**: Test builds without registry push
-
-### Build Variants
-
-Each build creates three image variants:
-
-- `rustfs/rustfs:v1.2.3` (production - Alpine-based)
-- `rustfs/rustfs:v1.2.3-source` (source build - Debian-based)
-- `rustfs/rustfs:v1.2.3-dev` (development - Debian-based with tools)
-
-### Manual Builds
-
-Trigger custom builds via GitHub Actions:
-
-```bash
-# Use workflow_dispatch to build specific versions
-# Available options: latest, main-latest, dev-latest, v1.2.3, dev-abc123
-```
-
-## 📦 Supporting Infrastructure
-
-The `.docker/` directory contains supporting configuration files:
-
-- **observability/** - Prometheus, Grafana, OpenTelemetry configs
-- **compose/** - Multi-service Docker Compose setups
-- **mqtt/** - MQTT broker configurations
-- **openobserve-otel/** - Log aggregation and tracing setup
-
-See individual README files in each subdirectory for specific usage instructions.

.docker/compose/README.md

@@ -1,80 +1,44 @@
-# Docker Compose Configurations
+# Specialized Docker Compose Configurations
 
-This directory contains specialized Docker Compose configurations for different use cases.
+This directory contains specialized Docker Compose configurations for specific testing scenarios.
+
+## ⚠️ Important Note
+
+**For Observability:**
+We **strongly recommend** using the new, fully integrated observability stack located in `../observability/`. It provides a production-ready setup with Prometheus, Grafana, Tempo, Loki, and OpenTelemetry Collector, all with persistent storage and optimized configurations.
+
+The `docker-compose.observability.yaml` in this directory is kept for legacy reference or specific minimal testing needs but is **not** the primary recommended setup.
 
 ## 📁 Configuration Files
 
-This directory contains specialized Docker Compose configurations and their associated Dockerfiles, keeping related files organized together.
+### Cluster Testing
 
-### Main Configuration (Root Directory)
+- **`docker-compose.cluster.yaml`**
+  - **Purpose**: Simulates a 4-node RustFS distributed cluster.
+  - **Use Case**: Testing distributed storage logic, consensus, and failover.
+  - **Nodes**: 4 RustFS instances.
+  - **Storage**: Uses local HTTP endpoints.
 
-- **`../../docker-compose.yml`** - **Default Production Setup**
-  - Complete production-ready configuration
-  - Includes RustFS server + full observability stack
-  - Supports multiple profiles: `dev`, `observability`, `cache`, `proxy`
-  - Recommended for most users
+### Legacy / Minimal Observability
 
-### Specialized Configurations
-
-- **`docker-compose.cluster.yaml`** - **Distributed Testing**
-  - 4-node cluster setup for testing distributed storage
-  - Uses local compiled binaries
-  - Simulates multi-node environment
-  - Ideal for development and cluster testing
-
-- **`docker-compose.observability.yaml`** - **Observability Focus**
-  - Specialized setup for testing observability features
-  - Includes OpenTelemetry, Jaeger, Prometheus, Loki, Grafana
-  - Uses `../../Dockerfile.source` for builds
-  - Perfect for observability development
+- **`docker-compose.observability.yaml`**
+  - **Purpose**: A minimal observability setup.
+  - **Status**: **Deprecated**. Please use `../observability/docker-compose.yml` instead.
 
 ## 🚀 Usage Examples
 
-### Production Setup
-
-```bash
-# Start main service
-docker-compose up -d
-
-# Start with development profile
-docker-compose --profile dev up -d
-
-# Start with full observability
-docker-compose --profile observability up -d
-```
-
 ### Cluster Testing
 
-```bash
-# Build and start 4-node cluster (run from project root)
-cd .docker/compose
-docker-compose -f docker-compose.cluster.yaml up -d
-
-# Or run directly from project root
-docker-compose -f .docker/compose/docker-compose.cluster.yaml up -d
-```
-
-### Observability Testing
+To start a 4-node cluster for distributed testing:
 
 ```bash
-# Start observability-focused environment (run from project root)
-cd .docker/compose
-docker-compose -f docker-compose.observability.yaml up -d
-
-# Or run directly from project root
-docker-compose -f .docker/compose/docker-compose.observability.yaml up -d
+# From project root
+docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
 ```
 
-## 🔧 Configuration Overview
+### (Deprecated) Minimal Observability
 
-| Configuration | Nodes | Storage | Observability | Use Case |
-|---------------|-------|---------|---------------|----------|
-| **Main** | 1 | Volume mounts | Full stack | Production |
-| **Cluster** | 4 | HTTP endpoints | Basic | Testing |
-| **Observability** | 4 | Local data | Advanced | Development |
-
-## 📝 Notes
-
-- Always ensure you have built the required binaries before starting cluster tests
-- The main configuration is sufficient for most use cases
-- Specialized configurations are for specific testing scenarios
+```bash
+# From project root
+docker compose -f .docker/compose/docker-compose.observability.yaml up -d
+```

.docker/compose/docker-compose.observability.yaml

@@ -13,65 +13,126 @@
 # limitations under the License.
 
 services:
+  # --- Observability Stack ---
+
+  tempo-init:
+    image: busybox:latest
+    command: [ "sh", "-c", "chown -R 10001:10001 /var/tempo" ]
+    volumes:
+      - tempo-data:/var/tempo
+    user: root
+    networks:
+      - rustfs-network
+    restart: "no"
+
+  tempo:
+    image: grafana/tempo:latest
+    user: "10001"
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ../../.docker/observability/tempo.yaml:/etc/tempo.yaml:ro
+      - tempo-data:/var/tempo
+    ports:
+      - "3200:3200" # tempo
+      - "4317"      # otlp grpc
+      - "4318"      # otlp http
+    restart: unless-stopped
+    networks:
+      - rustfs-network
+
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.129.1
+    image: otel/opentelemetry-collector-contrib:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
-      - ../../.docker/observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
+      - ../../.docker/observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
     ports:
-      - 1888:1888
-      - 8888:8888
-      - 8889:8889
-      - 13133:13133
-      - 4317:4317
-      - 4318:4318
-      - 55679:55679
+      - "1888:1888"   # pprof
+      - "8888:8888"   # Prometheus metrics for Collector
+      - "8889:8889"   # Prometheus metrics for application indicators
+      - "13133:13133" # health check
+      - "4317:4317"   # OTLP gRPC
+      - "4318:4318"   # OTLP HTTP
+      - "55679:55679" # zpages
     networks:
       - rustfs-network
+    depends_on:
+      - tempo
+      - jaeger
+      - prometheus
+      - loki
+
   jaeger:
-    image: jaegertracing/jaeger:2.8.0
+    image: jaegertracing/jaeger:latest
     environment:
       - TZ=Asia/Shanghai
+      - SPAN_STORAGE_TYPE=badger
+      - BADGER_EPHEMERAL=false
+      - BADGER_DIRECTORY_VALUE=/badger/data
+      - BADGER_DIRECTORY_KEY=/badger/key
+      - COLLECTOR_OTLP_ENABLED=true
+    volumes:
+      - jaeger-data:/badger
     ports:
-      - "16686:16686"
-      - "14317:4317"
-      - "14318:4318"
+      - "16686:16686" # Web UI
+      - "14269:14269" # Admin/Metrics
     networks:
       - rustfs-network
+
   prometheus:
-    image: prom/prometheus:v3.4.2
+    image: prom/prometheus:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
-      - ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
     ports:
       - "9090:9090"
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.enable-otlp-receiver'
+      - '--web.enable-remote-write-receiver'
+      - '--enable-feature=promql-experimental-functions'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
     networks:
       - rustfs-network
+
   loki:
-    image: grafana/loki:3.5.1
+    image: grafana/loki:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
-      - ../../.docker/observability/loki-config.yaml:/etc/loki/local-config.yaml
+      - ../../.docker/observability/loki-config.yaml:/etc/loki/local-config.yaml:ro
+      - loki-data:/loki
     ports:
       - "3100:3100"
     command: -config.file=/etc/loki/local-config.yaml
     networks:
       - rustfs-network
+
   grafana:
-    image: grafana/grafana:12.0.2
+    image: grafana/grafana:latest
     ports:
       - "3000:3000" # Web UI
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_SECURITY_ADMIN_USER=admin
       - TZ=Asia/Shanghai
+      - GF_INSTALL_PLUGINS=grafana-pyroscope-datasource
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/home.json
     networks:
       - rustfs-network
     volumes:
       - ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro
       - ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+    depends_on:
+      - prometheus
+      - tempo
+      - loki
+
+  # --- RustFS Cluster ---
 
   node1:
     build:
@@ -86,9 +147,11 @@ services:
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
-      - "9001:9000" # Map port 9001 of the host to port 9000 of the container
+      - "9001:9000"
     networks:
       - rustfs-network
+    depends_on:
+      - otel-collector
 
   node2:
     build:
@@ -103,9 +166,11 @@ services:
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
-      - "9002:9000" # Map port 9002 of the host to port 9000 of the container
+      - "9002:9000"
     networks:
       - rustfs-network
+    depends_on:
+      - otel-collector
 
   node3:
     build:
@@ -120,9 +185,11 @@ services:
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
-      - "9003:9000" # Map port 9003 of the host to port 9000 of the container
+      - "9003:9000"
     networks:
       - rustfs-network
+    depends_on:
+      - otel-collector
 
   node4:
     build:
@@ -137,9 +204,17 @@ services:
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
-      - "9004:9000" # Map port 9004 of the host to port 9000 of the container
+      - "9004:9000"
     networks:
       - rustfs-network
+    depends_on:
+      - otel-collector
+
+volumes:
+  prometheus-data:
+  tempo-data:
+  loki-data:
+  jaeger-data:
 
 networks:
   rustfs-network:

.docker/mqtt/README.md

@@ -0,0 +1,30 @@
+# MQTT Broker (EMQX)
+
+This directory contains the configuration for running an EMQX MQTT broker, which can be used for testing RustFS's MQTT integration.
+
+## 🚀 Quick Start
+
+To start the EMQX broker:
+
+```bash
+docker compose up -d
+```
+
+## 📊 Access
+
+- **Dashboard**: [http://localhost:18083](http://localhost:18083)
+- **Default Credentials**: `admin` / `public`
+- **MQTT Port**: `1883`
+- **WebSocket Port**: `8083`
+
+## 🛠️ Configuration
+
+The `docker-compose.yml` file sets up a single-node EMQX instance.
+
+- **Persistence**: Data is not persisted by default (for testing).
+- **Network**: Uses the default bridge network.
+
+## 📝 Notes
+
+- This setup is intended for development and testing purposes.
+- For production deployments, please refer to the official [EMQX Documentation](https://www.emqx.io/docs/en/latest/).

.docker/nginx/nginx.conf

@@ -0,0 +1,82 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+worker_processes auto;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+    error_log /var/log/nginx/error.log warn;
+
+    sendfile on;
+    keepalive_timeout 65;
+
+    # RustFS Server Block
+    server {
+        listen 80;
+        server_name localhost;
+
+        # Redirect HTTP to HTTPS (optional, uncomment if SSL is configured)
+        # return 301 https://$host$request_uri;
+
+        location / {
+            proxy_pass http://rustfs:9000;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            # S3 specific headers
+            proxy_set_header X-Amz-Date $http_x_amz_date;
+            proxy_set_header Authorization $http_authorization;
+
+            # Disable buffering for large uploads
+            proxy_request_buffering off;
+            client_max_body_size 0;
+        }
+
+        location /rustfs/console {
+            proxy_pass http://rustfs:9001;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+    }
+
+    # SSL Configuration (Example)
+    # server {
+    #     listen 443 ssl;
+    #     server_name localhost;
+    #
+    #     ssl_certificate /etc/nginx/ssl/server.crt;
+    #     ssl_certificate_key /etc/nginx/ssl/server.key;
+    #
+    #     location / {
+    #         proxy_pass http://rustfs:9000;
+    #         ...
+    #     }
+    # }
+}

.docker/observability/.gitignore

@@ -0,0 +1,5 @@
+jaeger-data/*
+loki-data/*
+prometheus-data/*
+tempo-data/*
+grafana-data/*

.docker/observability/README.md

@@ -1,109 +1,85 @@
-# Observability
+# RustFS Observability Stack
 
-This directory contains the observability stack for the application. The stack is composed of the following components:
+This directory contains the comprehensive observability stack for RustFS, designed to provide deep insights into application performance, logs, and traces.
 
-- Prometheus v3.2.1
-- Grafana 11.6.0
-- Loki 3.4.2
-- Jaeger 2.4.0
-- Otel Collector 0.120.0 # 0.121.0 remove loki
+## Components
 
-## Prometheus
+The stack is composed of the following best-in-class open-source components:
 
-Prometheus is a monitoring and alerting toolkit. It scrapes metrics from instrumented jobs, either directly or via an
-intermediary push gateway for short-lived jobs. It stores all scraped samples locally and runs rules over this data to
-either aggregate and record new time series from existing data or generate alerts. Grafana or other API consumers can be
-used to visualize the collected data.
+- **Prometheus** (v2.53.1): The industry standard for metric collection and alerting.
+- **Grafana** (v11.1.0): The leading platform for observability visualization.
+- **Loki** (v3.1.0): A horizontally-scalable, highly-available, multi-tenant log aggregation system.
+- **Tempo** (v2.5.0): A high-volume, minimal dependency distributed tracing backend.
+- **Jaeger** (v1.59.0): Distributed tracing system (configured as a secondary UI/storage).
+- **OpenTelemetry Collector** (v0.104.0): A vendor-agnostic implementation for receiving, processing, and exporting telemetry data.
 
-## Grafana
+## Architecture
 
-Grafana is a multi-platform open-source analytics and interactive visualization web application. It provides charts,
-graphs, and alerts for the web when connected to supported data sources.
+1. **Telemetry Collection**: Applications send OTLP (OpenTelemetry Protocol) data (Metrics, Logs, Traces) to the **OpenTelemetry Collector**.
+2. **Processing & Exporting**: The Collector processes the data (batching, memory limiting) and exports it to the respective backends:
+    - **Traces** -> **Tempo** (Primary) & **Jaeger** (Secondary/Optional)
+    - **Metrics** -> **Prometheus** (via scraping the Collector's exporter)
+    - **Logs** -> **Loki**
+3. **Visualization**: **Grafana** connects to all backends (Prometheus, Tempo, Loki, Jaeger) to provide a unified dashboard experience.
 
-## Loki
+## Features
 
-Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. It is
-designed to be very cost-effective and easy to operate. It does not index the contents of the logs, but rather a set of
-labels for each log stream.
+- **Full Persistence**: All data (Metrics, Logs, Traces) is persisted to Docker volumes, ensuring no data loss on restart.
+- **Correlation**: Seamless navigation between Metrics, Logs, and Traces in Grafana.
+  - Jump from a Metric spike to relevant Traces.
+  - Jump from a Trace to relevant Logs.
+- **High Performance**: Optimized configurations for batching, compression, and memory management.
+- **Standardized Protocols**: Built entirely on OpenTelemetry standards.
 
-## Jaeger
+## Quick Start
 
-Jaeger is a distributed tracing system released as open source by Uber Technologies. It is used for monitoring and
-troubleshooting microservices-based distributed systems, including:
+### Prerequisites
 
-- Distributed context propagation
-- Distributed transaction monitoring
-- Root cause analysis
-- Service dependency analysis
-- Performance / latency optimization
+- Docker
+- Docker Compose
 
-## Otel Collector
+### Deploy
 
-The OpenTelemetry Collector offers a vendor-agnostic implementation on how to receive, process, and export telemetry
-data. It removes the need to run, operate, and maintain multiple agents/collectors in order to support open-source
-observability data formats (e.g. Jaeger, Prometheus, etc.) sending to one or more open-source or commercial back-ends.
-
-## How to use
-
-To deploy the observability stack, run the following command:
-
-- docker latest version
+Run the following command to start the entire stack:
 
 ```bash
-docker compose -f docker-compose.yml -f docker-compose.override.yml up -d
+docker compose up -d
 ```
 
-- docker compose v2.0.0 or before
+### Access Dashboards
+
+| Service        | URL                                              | Credentials       | Description                    |
+| :------------- | :----------------------------------------------- | :---------------- | :----------------------------- |
+| **Grafana**    | [http://localhost:3000](http://localhost:3000)   | `admin` / `admin` | Main visualization hub.        |
+| **Prometheus** | [http://localhost:9090](http://localhost:9090)   | -                 | Metric queries and status.     |
+| **Jaeger UI**  | [http://localhost:16686](http://localhost:16686) | -                 | Secondary trace visualization. |
+| **Tempo**      | [http://localhost:3200](http://localhost:3200)   | -                 | Tempo status/metrics.          |
+
+## Configuration
+
+### Data Persistence
+
+Data is stored in the following Docker volumes:
+
+- `prometheus-data`: Prometheus metrics
+- `tempo-data`: Tempo traces (WAL and Blocks)
+- `loki-data`: Loki logs (Chunks and Rules)
+- `jaeger-data`: Jaeger traces (Badger DB)
+
+To clear all data:
 
 ```bash
-docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d
+docker compose down -v
 ```
 
-To access the Grafana dashboard, navigate to `http://localhost:3000` in your browser. The default username and password
-are `admin` and `admin`, respectively.
-
-To access the Jaeger dashboard, navigate to `http://localhost:16686` in your browser.
-
-To access the Prometheus dashboard, navigate to `http://localhost:9090` in your browser.
-
-## How to stop
-
-To stop the observability stack, run the following command:
-
-```bash
-docker compose -f docker-compose.yml -f docker-compose.override.yml down
-```
-
-## How to remove data
-
-To remove the data generated by the observability stack, run the following command:
-
-```bash
-docker compose -f docker-compose.yml -f docker-compose.override.yml down -v
-```
-
-## How to configure
-
-To configure the observability stack, modify the `docker-compose.override.yml` file. The file contains the following
-
-```yaml
-services:
-  prometheus:
-    environment:
-      - PROMETHEUS_CONFIG_FILE=/etc/prometheus/prometheus.yml
-    volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
-
-  grafana:
-    environment:
-      - GF_SECURITY_ADMIN_PASSWORD=admin
-    volumes:
-      - ./grafana/provisioning:/etc/grafana/provisioning
-```
-
-The `prometheus` service mounts the `prometheus.yml` file to `/etc/prometheus/prometheus.yml`. The `grafana` service
-mounts the `grafana/provisioning` directory to `/etc/grafana/provisioning`. You can modify these files to configure the
-observability stack.
+### Customization
 
+- **Prometheus**: Edit `prometheus.yml` to add scrape targets or alerting rules.
+- **Grafana**: Dashboards and datasources are provisioned from the `grafana/` directory.
+- **Collector**: Edit `otel-collector-config.yaml` to modify pipelines, processors, or exporters.
 
+## Troubleshooting
 
+- **Service Health**: Check the health of services using `docker compose ps`.
+- **Logs**: View logs for a specific service using `docker compose logs -f <service_name>`.
+- **Otel Collector**: Check `http://localhost:13133` for health status and `http://localhost:1888/debug/pprof/` for profiling.

.docker/observability/README_ZH.md

@@ -1,27 +1,85 @@
-## 部署可观测性系统
+# RustFS 可观测性技术栈
 
-OpenTelemetry Collector 提供了一个厂商中立的遥测数据处理方案，用于接收、处理和导出遥测数据。它消除了为支持多种开源可观测性数据格式（如
-Jaeger、Prometheus 等）而需要运行和维护多个代理/收集器的必要性。
+本目录包含 RustFS 的全面可观测性技术栈，旨在提供对应用程序性能、日志和追踪的深入洞察。
 
-### 快速部署
+## 组件
 
-1. 进入 `.docker/observability` 目录
-2. 执行以下命令启动服务：
+该技术栈由以下一流的开源组件组成：
+
+- **Prometheus** (v2.53.1): 行业标准的指标收集和告警工具。
+- **Grafana** (v11.1.0): 领先的可观测性可视化平台。
+- **Loki** (v3.1.0): 水平可扩展、高可用、多租户的日志聚合系统。
+- **Tempo** (v2.5.0): 高吞吐量、最小依赖的分布式追踪后端。
+- **Jaeger** (v1.59.0): 分布式追踪系统（配置为辅助 UI/存储）。
+- **OpenTelemetry Collector** (v0.104.0): 接收、处理和导出遥测数据的供应商无关实现。
+
+## 架构
+
+1.  **遥测收集**: 应用程序将 OTLP (OpenTelemetry Protocol) 数据（指标、日志、追踪）发送到 **OpenTelemetry Collector**。
+2.  **处理与导出**: Collector 处理数据（批处理、内存限制）并将其导出到相应的后端：
+    -   **追踪** -> **Tempo** (主要) & **Jaeger** (辅助/可选)
+    -   **指标** -> **Prometheus** (通过抓取 Collector 的导出器)
+    -   **日志** -> **Loki**
+3.  **可视化**: **Grafana** 连接到所有后端（Prometheus, Tempo, Loki, Jaeger），提供统一的仪表盘体验。
+
+## 特性
+
+-   **完全持久化**: 所有数据（指标、日志、追踪）都持久化到 Docker 卷，确保重启后无数据丢失。
+-   **关联性**: 在 Grafana 中实现指标、日志和追踪之间的无缝导航。
+    -   从指标峰值跳转到相关追踪。
+    -   从追踪跳转到相关日志。
+-   **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
+-   **标准化协议**: 完全基于 OpenTelemetry 标准构建。
+
+## 快速开始
+
+### 前置条件
+
+-   Docker
+-   Docker Compose
+
+### 部署
+
+运行以下命令启动整个技术栈：
 
 ```bash
-docker compose -f docker-compose.yml  up -d
+docker compose up -d
 ```
 
-### 访问监控面板
+### 访问仪表盘
 
-服务启动后，可通过以下地址访问各个监控面板：
+| 服务 | URL | 凭据 | 描述 |
+| :--- | :--- | :--- | :--- |
+| **Grafana** | [http://localhost:3000](http://localhost:3000) | `admin` / `admin` | 主要可视化中心。 |
+| **Prometheus** | [http://localhost:9090](http://localhost:9090) | - | 指标查询和状态。 |
+| **Jaeger UI** | [http://localhost:16686](http://localhost:16686) | - | 辅助追踪可视化。 |
+| **Tempo** | [http://localhost:3200](http://localhost:3200) | - | Tempo 状态/指标。 |
 
-- Grafana: `http://localhost:3000` (默认账号/密码：`admin`/`admin`)
-- Jaeger: `http://localhost:16686`
-- Prometheus: `http://localhost:9090`
+## 配置
 
-## 配置可观测性
+### 数据持久化
 
-```shell
-export RUSTFS_OBS_ENDPOINT="http://localhost:4317" # OpenTelemetry Collector 地址
+数据存储在以下 Docker 卷中：
+
+-   `prometheus-data`: Prometheus 指标
+-   `tempo-data`: Tempo 追踪 (WAL 和 Blocks)
+-   `loki-data`: Loki 日志 (Chunks 和 Rules)
+-   `jaeger-data`: Jaeger 追踪 (Badger DB)
+
+要清除所有数据：
+
+```bash
+docker compose down -v
 ```
+
+### 自定义
+
+-   **Prometheus**: 编辑 `prometheus.yml` 以添加抓取目标或告警规则。
+-   **Grafana**: 仪表盘和数据源从 `grafana/` 目录预置。
+-   **Collector**: 编辑 `otel-collector-config.yaml` 以修改管道、处理器或导出器。
+
+## 故障排除
+
+-   **服务健康**: 使用 `docker compose ps` 检查服务健康状况。
+-   **日志**: 使用 `docker compose logs -f <service_name>` 查看特定服务的日志。
+-   **Otel Collector**: 检查 `http://localhost:13133` 获取健康状态，检查 `http://localhost:1888/debug/pprof/` 进行性能分析。

.docker/observability/docker-compose.yml

@@ -13,164 +13,241 @@
 # limitations under the License.
 
 services:
-
-  tempo-init:
-    image: busybox:latest
-    command: [ "sh", "-c", "chown -R 10001:10001 /var/tempo" ]
+  rustfs:
+    security_opt:
+      - "no-new-privileges:true"
+    image: rustfs/rustfs:latest
+    container_name: rustfs-server
+    ports:
+      - "9000:9000" # S3 API port
+      - "9001:9001" # Console port
+    environment:
+      - RUSTFS_VOLUMES=/data/rustfs
+      - RUSTFS_ADDRESS=0.0.0.0:9000
+      - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_CORS_ALLOWED_ORIGINS=*
+      - RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS=*
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
+      - RUSTFS_OBS_LOGGER_LEVEL=info
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
     volumes:
-      - ./tempo-data:/var/tempo
-    user: root
+      - rustfs-data:/data/rustfs
     networks:
       - otel-network
-    restart: "no"
+    restart: unless-stopped
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "sh",
+          "-c",
+          "curl -f http://127.0.0.1:9000/health && curl -f http://127.0.0.1:9001/rustfs/console/health",
+        ]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    depends_on:
+      otel-collector:
+        condition: service_started
+
+  rustfs-init:
+    image: alpine
+    container_name: rustfs-init
+    volumes:
+      - rustfs-data:/data
+    networks:
+      - otel-network
+    command: >
+      sh -c "
+        chown -R 10001:10001 /data &&
+        echo 'Volume Permissions fixed' &&
+        exit 0
+      "
+    restart: no
+
+  # --- Tracing ---
 
   tempo:
     image: grafana/tempo:latest
-    user: "10001" # The container must be started with root to execute chown in the script
-    command: [ "-config.file=/etc/tempo.yaml" ] # This is passed as a parameter to the entry point script
+    container_name: tempo
+    command: ["-config.file=/etc/tempo.yaml"]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
-      - ./tempo-data:/var/tempo
+      - tempo-data:/var/tempo
     ports:
       - "3200:3200" # tempo
-      - "24317:4317" # otlp grpc
-      - "24318:4318" # otlp http
-    restart: unless-stopped
+      - "4317"      # otlp grpc
+      - "4318"      # otlp http
     networks:
       - otel-network
+    restart: unless-stopped
+    depends_on:
+      - redpanda
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/metrics" ]
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3200/ready"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 15s
 
-  otel-collector:
-    image: otel/opentelemetry-collector-contrib:latest
-    environment:
-      - TZ=Asia/Shanghai
-    volumes:
-      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
+  redpanda:
+    image: redpandadata/redpanda:latest # for tempo ingest
+    container_name: redpanda
     ports:
-      - "1888:1888" # pprof
-      - "8888:8888" # Prometheus metrics for Collector
-      - "8889:8889" # Prometheus metrics for application indicators
-      - "13133:13133" # health check
-      - "4317:4317" # OTLP gRPC
-      - "4318:4318" # OTLP HTTP
-      - "55679:55679" # zpages
+      - "9092:9092"
     networks:
       - otel-network
-    depends_on:
-      jaeger:
-        condition: service_started
-      tempo:
-        condition: service_started
-      prometheus:
-        condition: service_started
-      loki:
-        condition: service_started
+    restart: unless-stopped
+    command: >
+      redpanda start --overprovisioned
+      --mode=dev-container
+      --kafka-addr=PLAINTEXT://0.0.0.0:9092
+      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
+
+  jaeger:
+    image: jaegertracing/jaeger:latest
+    container_name: jaeger
+    environment:
+      - SPAN_STORAGE_TYPE=badger
+      - BADGER_EPHEMERAL=false
+      - BADGER_DIRECTORY_VALUE=/badger/data
+      - BADGER_DIRECTORY_KEY=/badger/key
+      - COLLECTOR_OTLP_ENABLED=true
+    volumes:
+      - ./jaeger.yaml:/etc/jaeger/config.yml
+      - jaeger-data:/badger
+    ports:
+      - "16686:16686" # Web UI
+      - "14269:14269" # Admin/Metrics
+      - "4317" # otlp grpc
+      - "4318" # otlp http
+    command: ["--config", "/etc/jaeger/config.yml"]
+    networks:
+      - otel-network
+    restart: unless-stopped
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:14269"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
+
+  # --- Metrics ---
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    ports:
+      - "9090:9090"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--web.enable-otlp-receiver" # Enable OTLP
+      - "--web.enable-remote-write-receiver" # Enable remote write
+      - "--enable-feature=promql-experimental-functions" # Enable info()
+      - "--storage.tsdb.retention.time=30d"
+    restart: unless-stopped
+    networks:
+      - otel-network
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"]
       interval: 10s
       timeout: 5s
       retries: 3
 
-  jaeger:
-    image: jaegertracing/jaeger:latest
-    environment:
-      - TZ=Asia/Shanghai
-      - SPAN_STORAGE_TYPE=memory
-      - COLLECTOR_OTLP_ENABLED=true
-    ports:
-      - "16686:16686" # Web UI
-      - "14317:4317" # OTLP gRPC
-      - "14318:4318" # OTLP HTTP
-      - "18888:8888" # collector
-    networks:
-      - otel-network
-    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:16686" ]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-  prometheus:
-    image: prom/prometheus:latest
-    environment:
-      - TZ=Asia/Shanghai
-    volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - ./prometheus-data:/prometheus
-    ports:
-      - "9090:9090"
-    command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--web.enable-otlp-receiver' # Enable OTLP
-      - '--web.enable-remote-write-receiver' # Enable remote write
-      - '--enable-feature=promql-experimental-functions' # Enable info()
-      - '--storage.tsdb.min-block-duration=15m' # Minimum block duration
-      - '--storage.tsdb.max-block-duration=1h'  # Maximum block duration
-      - '--log.level=info'
-      - '--storage.tsdb.retention.time=30d'
-      - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
-    restart: unless-stopped
-    networks:
-      - otel-network
-    healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
-      interval: 10s
-      timeout: 5s
-      retries: 3
+  # --- Logging ---
+
   loki:
     image: grafana/loki:latest
-    environment:
-      - TZ=Asia/Shanghai
+    container_name: loki
     volumes:
-      - ./loki-config.yaml:/etc/loki/local-config.yaml:ro
+      - ./loki.yaml:/etc/loki/loki.yaml:ro
+      - loki-data:/loki
     ports:
       - "3100:3100"
-    command: -config.file=/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/loki.yaml
     networks:
       - otel-network
+    restart: unless-stopped
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3100/ready"]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+
+  # --- Collection ---
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
+    ports:
+      - "1888:1888"   # pprof
+      - "8888:8888"   # Prometheus metrics for Collector
+      - "8889:8889"   # Prometheus metrics for application indicators
+      - "13133:13133" # health check
+      - "4317:4317"   # OTLP gRPC
+      - "4318:4318"   # OTLP HTTP
+      - "55679:55679" # zpages
+    networks:
+      - otel-network
+    restart: unless-stopped
+    depends_on:
+      - tempo
+      - jaeger
+      - prometheus
+      - loki
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:13133"]
       interval: 10s
       timeout: 5s
       retries: 3
+
+  # --- Visualization ---
+
   grafana:
     image: grafana/grafana:latest
+    container_name: grafana
     ports:
-      - "3000:3000"  # Web UI
-    volumes:
-      - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+      - "3000:3000"
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_SECURITY_ADMIN_USER=admin
-      - TZ=Asia/Shanghai
-      - GF_INSTALL_PLUGINS=grafana-pyroscope-datasource
-    restart: unless-stopped
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/etc/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
     networks:
       - otel-network
+    restart: unless-stopped
     depends_on:
       - prometheus
       - tempo
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"]
       interval: 10s
       timeout: 5s
       retries: 3
 
 volumes:
-  prometheus-data:
+  rustfs-data:
   tempo-data:
+  jaeger-data:
+  prometheus-data:
+  loki-data:
+  grafana-data:
 
 networks:
   otel-network:
     driver: bridge
-    name: "network_otel_config"
+    name: "network_otel"
     ipam:
       config:
         - subnet: 172.28.0.0/16

.docker/observability/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: "default"
+    orgId: 1
+    folder: ""
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    options:
+      path: /etc/grafana/dashboards

.docker/observability/grafana/provisioning/datasources.yaml

@@ -1,3 +1,17 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 apiVersion: 1
 
 datasources:
@@ -7,102 +21,77 @@ datasources:
     access: proxy
     orgId: 1
     url: http://prometheus:9090
-    basicAuth: false
-    isDefault: false
+    isDefault: true
     version: 1
     editable: false
     jsonData:
       httpMethod: GET
+      exemplarTraceIdDestinations:
+        - name: trace_id
+          datasourceUid: tempo
+
   - name: Tempo
     type: tempo
+    uid: tempo
     access: proxy
     orgId: 1
     url: http://tempo:3200
-    basicAuth: false
-    isDefault: true
+    isDefault: false
     version: 1
     editable: false
-    apiVersion: 1
-    uid: tempo
     jsonData:
       httpMethod: GET
       serviceMap:
         datasourceUid: prometheus
-      streamingEnabled:
-        search: true
-      tracesToLogsV2:
-        # Field with an internal link pointing to a logs data source in Grafana.
-        # datasourceUid value must match the uid value of the logs data source.
-        datasourceUid: 'loki'
-        spanStartTimeShift: '-1h'
-        spanEndTimeShift: '1h'
-        tags: [ 'job', 'instance', 'pod', 'namespace' ]
-        filterByTraceID: false
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '1s'
+        spanEndTimeShift: '-1s'
+        filterByTraceID: true
         filterBySpanID: false
-        customQuery: true
-        query: 'method="$${__span.tags.method}"'
-        tracesToMetrics:
-          datasourceUid: 'prometheus'
-          spanStartTimeShift: '-1h'
-          spanEndTimeShift: '1h'
-          tags: [ { key: 'service.name', value: 'service' }, { key: 'job' } ]
-          queries:
-            - name: 'Sample query'
-              query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
-        tracesToProfiles:
-          datasourceUid: 'grafana-pyroscope-datasource'
-          tags: [ 'job', 'instance', 'pod', 'namespace' ]
-          profileTypeId: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds'
-          customQuery: true
-          query: 'method="$${__span.tags.method}"'
-        serviceMap:
-          datasourceUid: 'prometheus'
-        nodeGraph:
-          enabled: true
-        search:
-          hide: false
-        traceQuery:
-          timeShiftEnabled: true
-          spanStartTimeShift: '-1h'
-          spanEndTimeShift: '1h'
-        spanBar:
-          type: 'Tag'
-          tag: 'http.path'
-        streamingEnabled:
-          search: true
-  - name: Jaeger
-    type: jaeger
-    uid: Jaeger
-    url: http://jaeger:16686
-    basicAuth: false
-    access: proxy
-    readOnly: false
-    isDefault: false
-    jsonData:
-      tracesToLogsV2:
-        # Field with an internal link pointing to a logs data source in Grafana.
-        # datasourceUid value must match the uid value of the logs data source.
-        datasourceUid: 'loki'
-        spanStartTimeShift: '1h'
-        spanEndTimeShift: '-1h'
-        tags: [ 'job', 'instance', 'pod', 'namespace' ]
-        filterByTraceID: false
-        filterBySpanID: false
-        customQuery: true
-        query: 'method="$${__span.tags.method}"'
       tracesToMetrics:
-        datasourceUid: 'Prometheus'
-        spanStartTimeShift: '1h'
-        spanEndTimeShift: '-1h'
-        tags: [ { key: 'service.name', value: 'service' }, { key: 'job' } ]
+        datasourceUid: prometheus
+        tags: [ { key: 'service.name' }, { key: 'job' } ]
         queries:
-          - name: 'Sample query'
-            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
+          - name: 'Service-Level Latency'
+            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
+          - name: 'Service-Level Calls'
+            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
+          - name: 'Service-Level Errors'
+            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
       nodeGraph:
         enabled: true
-      traceQuery:
-        timeShiftEnabled: true
-        spanStartTimeShift: '1h'
-        spanEndTimeShift: '-1h'
-      spanBar:
-        type: 'None'
+
+  - name: Loki
+    type: loki
+    uid: loki
+    orgId: 1
+    url: http://loki:3100
+    isDefault: false
+    version: 1
+    editable: false
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: 'trace_id=(\w+)'
+          name: 'TraceID'
+          url: '$${__value.raw}'
+
+  - name: Jaeger
+    type: jaeger
+    uid: jaeger
+    url: http://jaeger:16686
+    access: proxy
+    isDefault: false
+    editable: false
+    jsonData:
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '1s'
+        spanEndTimeShift: '-1s'
+        filterByTraceID: true
+        filterBySpanID: false

.docker/observability/grafana/provisioning/datasources/datasources.yaml

@@ -0,0 +1,78 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    url: http://prometheus:9090
+    access: proxy
+    isDefault: true
+    editable: false
+    jsonData:
+      httpMethod: GET
+      exemplarTraceIdDestinations:
+        - name: trace_id
+          datasourceUid: tempo
+
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    access: proxy
+    url: http://tempo:3200
+    isDefault: false
+    editable: false
+    jsonData:
+      httpMethod: GET
+      serviceMap:
+        datasourceUid: prometheus
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '-1h'
+        spanEndTimeShift: '1h'
+        filterByTraceID: true
+        filterBySpanID: false
+      tracesToMetrics:
+        datasourceUid: prometheus
+        tags: [ { key: 'service.name' }, { key: 'job' } ]
+        queries:
+          - name: 'Service-Level Latency'
+            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
+          - name: 'Service-Level Calls'
+            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
+          - name: 'Service-Level Errors'
+            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
+      nodeGraph:
+        enabled: true
+
+  - name: Loki
+    type: loki
+    uid: loki
+    url: http://loki:3100
+    basicAuth: false
+    isDefault: false
+    editable: false
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: 'trace_id=(\w+)'
+          name: 'TraceID'
+          url: '$${__value.raw}'
+
+  - name: Jaeger
+    type: jaeger
+    uid: jaeger
+    url: http://jaeger:16686
+    access: proxy
+    isDefault: false
+    editable: false
+    jsonData:
+      tracesToLogs:
+        datasourceUid: loki
+        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
+        mappedTags: [ { key: 'service.name', value: 'app' } ]
+        spanStartTimeShift: '1s'
+        spanEndTimeShift: '-1s'
+        filterByTraceID: true
+        filterBySpanID: false

.docker/observability/jaeger-config.yaml

@@ -1,114 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-service:
-  extensions: [ jaeger_storage, jaeger_query, remote_sampling, healthcheckv2 ]
-  pipelines:
-    traces:
-      receivers: [ otlp, jaeger, zipkin ]
-      processors: [ batch, adaptive_sampling ]
-      exporters: [ jaeger_storage_exporter ]
-  telemetry:
-    resource:
-      service.name: jaeger
-    metrics:
-      level: detailed
-      readers:
-        - pull:
-            exporter:
-              prometheus:
-                host: 0.0.0.0
-                port: 8888
-    logs:
-      level: debug
-    # TODO Initialize telemetry tracer once OTEL released new feature.
-    # https://github.com/open-telemetry/opentelemetry-collector/issues/10663
-
-extensions:
-  healthcheckv2:
-    use_v2: true
-    http:
-
-  # pprof:
-  #   endpoint: 0.0.0.0:1777
-  # zpages:
-  #   endpoint: 0.0.0.0:55679
-
-  jaeger_query:
-    storage:
-      traces: some_store
-      traces_archive: another_store
-    ui:
-      config_file: ./cmd/jaeger/config-ui.json
-      log_access: true
-    # The maximum duration that is considered for clock skew adjustments.
-    # Defaults to 0 seconds, which means it's disabled.
-    max_clock_skew_adjust: 0s
-    grpc:
-      endpoint: 0.0.0.0:16685
-    http:
-      endpoint: 0.0.0.0:16686
-
-  jaeger_storage:
-    backends:
-      some_store:
-        memory:
-          max_traces: 1000000
-          max_events: 100000
-      another_store:
-        memory:
-          max_traces: 1000000
-    metric_backends:
-      some_metrics_storage:
-        prometheus:
-          endpoint: http://prometheus:9090
-          normalize_calls: true
-          normalize_duration: true
-
-  remote_sampling:
-    # You can either use file or adaptive sampling strategy in remote_sampling
-    # file:
-    #   path: ./cmd/jaeger/sampling-strategies.json
-    adaptive:
-      sampling_store: some_store
-      initial_sampling_probability: 0.1
-    http:
-    grpc:
-
-receivers:
-  otlp:
-    protocols:
-      grpc:
-      http:
-
-  jaeger:
-    protocols:
-      grpc:
-      thrift_binary:
-      thrift_compact:
-      thrift_http:
-
-  zipkin:
-
-processors:
-  batch:
-  metadata_keys: [ "span.kind", "http.method", "http.status_code", "db.system", "db.statement", "messaging.system", "messaging.destination", "messaging.operation","span.events","span.links" ]
-  # Adaptive Sampling Processor is required to support adaptive sampling.
-  # It expects remote_sampling extension with `adaptive:` config to be enabled.
-  adaptive_sampling:
-
-exporters:
-  jaeger_storage_exporter:
-    trace_storage: some_store
-

.docker/observability/jaeger.yaml

@@ -0,0 +1,74 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+service:
+  extensions: [jaeger_storage, jaeger_query]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [jaeger_storage_exporter, spanmetrics]
+    metrics/spanmetrics:
+      receivers: [spanmetrics]
+      exporters: [prometheus]
+  telemetry:
+    resource:
+      service.name: jaeger
+    metrics:
+      level: detailed
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: 0.0.0.0
+                port: 8888
+    logs:
+      level: DEBUG
+
+extensions:
+  jaeger_query:
+    storage:
+      traces: some_storage
+      metrics: some_metrics_storage
+  jaeger_storage:
+    backends:
+      some_storage:
+        memory:
+          max_traces: 100000
+    metric_backends:
+      some_metrics_storage:
+        prometheus:
+          endpoint: http://prometheus:9090
+          normalize_calls: true
+          normalize_duration: true
+
+connectors:
+  spanmetrics:
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "0.0.0.0:4317"
+      http:
+        endpoint: "0.0.0.0:4318"
+
+processors:
+  batch:
+
+exporters:
+  jaeger_storage_exporter:
+    trace_storage: some_storage
+  prometheus:
+    endpoint: "0.0.0.0:8889"

.docker/observability/loki.yaml

@@ -11,22 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 auth_enabled: false
 
 server:
   http_listen_port: 3100
-  grpc_listen_port: 9096
-  log_level: debug
+  grpc_listen_port: 9095
+  log_level: info
   grpc_server_max_concurrent_streams: 1000
 
 common:
   instance_addr: 127.0.0.1
-  path_prefix: /tmp/loki
+  path_prefix: /loki
   storage:
     filesystem:
-      chunks_directory: /tmp/loki/chunks
-      rules_directory: /tmp/loki/rules
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
   replication_factor: 1
   ring:
     kvstore:
@@ -39,12 +38,6 @@ query_range:
         enabled: true
         max_size_mb: 100
 
-limits_config:
-  metric_aggregation_enabled: true
-  max_line_size: 256KB
-  max_line_size_truncate: false
-  allow_structured_metadata: true
-
 schema_config:
   configs:
     - from: 2020-10-24
@@ -54,29 +47,17 @@ schema_config:
       index:
         prefix: index_
         period: 24h
-      row_shards: 16
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  allow_structured_metadata: true
+  max_line_size: 256KB
 
 pattern_ingester:
   enabled: true
   metric_aggregation:
     loki_address: localhost:3100
 
-ruler:
-  alertmanager_url: http://localhost:9093
-
 frontend:
   encoding: protobuf
-
-
-# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
-# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
-#
-# Statistics help us better understand how Loki is used, and they show us performance
-# levels for most users. This helps us prioritize features and documentation.
-# For more information on what's sent, look at
-# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
-# Refer to the buildReport method to see what goes into a report.
-#
-# If you would like to disable reporting, uncomment the following lines:
-#analytics:
-#  reporting_enabled: false

.docker/observability/otel-collector-config.yaml

@@ -15,69 +15,70 @@
 receivers:
   otlp:
     protocols:
-      grpc: # OTLP gRPC receiver
+      grpc:
         endpoint: 0.0.0.0:4317
-      http: # OTLP HTTP receiver
+      http:
         endpoint: 0.0.0.0:4318
 
 processors:
-  batch: # Batch processor to improve throughput
-    timeout: 5s
-    send_batch_size: 1000
-    metadata_keys: [ ]
-    metadata_cardinality_limit: 1000
+  batch:
+    timeout: 1s
+    send_batch_size: 1024
   memory_limiter:
     check_interval: 1s
-    limit_mib: 512
+    limit_mib: 1024
+    spike_limit_mib: 256
   transform/logs:
     log_statements:
       - context: log
         statements:
-          # Extract Body as attribute "message"
           - set(attributes["message"], body.string)
-          # Retain the original Body
           - set(attributes["log.body"], body.string)
 
 exporters:
-  otlp/traces: # OTLP exporter for trace data
-    endpoint: "http://jaeger:4317"  # OTLP gRPC endpoint for Jaeger
+  otlp/tempo:
+    endpoint: "tempo:4317"
     tls:
-      insecure: true  # TLS is disabled in the development environment and a certificate needs to be configured in the production environment.
-    compression: gzip  # Enable compression to reduce network bandwidth
+      insecure: true
+    compression: gzip
     retry_on_failure:
-      enabled: true  # Enable retry on failure
-      initial_interval: 1s  # Initial interval for retry
-      max_interval: 30s  # Maximum interval for retry
-      max_elapsed_time: 300s  # Maximum elapsed time for retry
+      enabled: true
+      initial_interval: 1s
+      max_interval: 30s
+      max_elapsed_time: 300s
     sending_queue:
-      enabled: true  # Enable sending queue
-      num_consumers: 10  # Number of consumers
-      queue_size: 5000  # Queue size
-  otlp/tempo: # OTLP exporter for trace data
-    endpoint: "http://tempo:4317"  # OTLP gRPC endpoint for tempo
+      enabled: true
+      num_consumers: 10
+      queue_size: 5000
+
+  otlp/jaeger:
+    endpoint: "jaeger:4317"
     tls:
-      insecure: true  # TLS is disabled in the development environment and a certificate needs to be configured in the production environment.
-    compression: gzip  # Enable compression to reduce network bandwidth
+      insecure: true
+    compression: gzip
     retry_on_failure:
-      enabled: true  # Enable retry on failure
-      initial_interval: 1s  # Initial interval for retry
-      max_interval: 30s  # Maximum interval for retry
-      max_elapsed_time: 300s  # Maximum elapsed time for retry
+      enabled: true
+      initial_interval: 1s
+      max_interval: 30s
+      max_elapsed_time: 300s
     sending_queue:
-      enabled: true  # Enable sending queue
-      num_consumers: 10  # Number of consumers
-      queue_size: 5000  # Queue size
-  prometheus: # Prometheus exporter for metrics data
-    endpoint: "0.0.0.0:8889"  # Prometheus scraping endpoint
-    send_timestamps: true  # Send timestamp
-    metric_expiration: 5m  # Metric expiration time
+      enabled: true
+      num_consumers: 10
+      queue_size: 5000
+
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+    send_timestamps: true
+    metric_expiration: 5m
     resource_to_telemetry_conversion:
-      enabled: true  # Enable resource to telemetry conversion
-  otlphttp/loki: # Loki exporter for log data
+      enabled: true
+
+  otlphttp/loki:
     endpoint: "http://loki:3100/otlp"
     tls:
       insecure: true
-    compression: gzip  # Enable compression to reduce network bandwidth
+    compression: gzip
+
 extensions:
   health_check:
     endpoint: 0.0.0.0:13133
@@ -85,13 +86,14 @@ extensions:
     endpoint: 0.0.0.0:1888
   zpages:
     endpoint: 0.0.0.0:55679
+
 service:
-  extensions: [ health_check, pprof, zpages ]  # Enable extension
+  extensions: [ health_check, pprof, zpages ]
   pipelines:
     traces:
       receivers: [ otlp ]
       processors: [ memory_limiter, batch ]
-      exporters: [ otlp/traces, otlp/tempo ]
+      exporters: [ otlp/tempo, otlp/jaeger ]
     metrics:
       receivers: [ otlp ]
       processors: [ batch ]
@@ -102,20 +104,13 @@ service:
       exporters: [ otlphttp/loki ]
   telemetry:
     logs:
-      level: "debug"  # Collector log level
-      encoding: "json"  # Log encoding: console or json
+      level: "info"
+      encoding: "json"
     metrics:
-      level: "detailed" # Can be basic, normal, detailed
+      level: "normal"
       readers:
-        - periodic:
-            exporter:
-              otlp:
-                protocol: http/protobuf
-                endpoint: http://otel-collector:4318
         - pull:
             exporter:
               prometheus:
                 host: '0.0.0.0'
                 port: 8888
-
-

.docker/observability/prometheus-data/.gitignore

@@ -1 +0,0 @@
-*

.docker/observability/prometheus.yml

@@ -17,27 +17,40 @@ global:
   evaluation_interval: 15s
   external_labels:
     cluster: 'rustfs-dev'  # Label to identify the cluster
-    relica: '1'  # Replica identifier
+    replica: '1'  # Replica identifier
 
 scrape_configs:
-  - job_name: 'otel-collector-internal'
+  - job_name: 'otel-collector'
     static_configs:
       - targets: [ 'otel-collector:8888' ]  # Scrape metrics from Collector
     scrape_interval: 10s
+
   - job_name: 'rustfs-app-metrics'
     static_configs:
       - targets: [ 'otel-collector:8889' ]  # Application indicators
     scrape_interval: 15s
     metric_relabel_configs:
+      - source_labels: [ __name__ ]
+        regex: 'go_.*'
+        action: drop  # Drop Go runtime metrics if not needed
+
   - job_name: 'tempo'
     static_configs:
       - targets: [ 'tempo:3200' ]  # Scrape metrics from Tempo
+
   - job_name: 'jaeger'
     static_configs:
-      - targets: [ 'jaeger:8888' ]  # Jaeger admin port
+      - targets: [ 'jaeger:14269' ]  # Jaeger admin port (14269 is standard for admin/metrics)
+
+  - job_name: 'loki'
+    static_configs:
+      - targets: [ 'loki:3100' ]
+
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: [ 'localhost:9090' ]
 
 otlp:
-  # Recommended attributes to be promoted to labels.
   promote_resource_attributes:
     - service.instance.id
     - service.name
@@ -56,10 +69,8 @@ otlp:
     - k8s.pod.name
     - k8s.replicaset.name
     - k8s.statefulset.name
-  # Ingest OTLP data keeping all characters in metric/label names.
   translation_strategy: NoUTF8EscapingWithSuffixes
 
 storage:
-  # OTLP is a push-based protocol, Out of order samples is a common scenario.
   tsdb:
-    out_of_order_time_window: 30m
+    out_of_order_time_window: 30m

.docker/observability/tempo-data/.gitignore

@@ -1 +0,0 @@
-*

.docker/observability/tempo.yaml

@@ -1,33 +1,55 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+partition_ring_live_store: true
 stream_over_http_enabled: true
+
 server:
   http_listen_port: 3200
   log_level: info
 
-query_frontend:
-  search:
-    duration_slo: 5s
-    throughput_bytes_slo: 1.073741824e+09
-    metadata_slo:
-      duration_slo: 5s
-      throughput_bytes_slo: 1.073741824e+09
-  trace_by_id:
-    duration_slo: 5s
-
 distributor:
+  ingester_write_path_enabled: false
+  kafka_write_path_enabled: true
   receivers:
     otlp:
       protocols:
         grpc:
-          endpoint: "0.0.0.0:4317"
+          endpoint: "tempo:4317"
         http:
-          endpoint: "0.0.0.0:4318"
+          endpoint: "tempo:4318"
+  #log_received_spans:
+  #    enabled: true
+  #  log_discarded_spans:
+  #    enabled: true
 
-ingester:
-  max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+backend_scheduler:
+  provider:
+    compaction:
+      compaction:
+        block_retention: 1h
 
-compactor:
+backend_worker:
+  backend_scheduler_addr: localhost:3200
   compaction:
-    block_retention: 1h # overall Tempo trace retention. set for demo purposes
+    block_retention: 1h
+  ring:
+    kvstore:
+      store: memberlist
+
+querier:
+  query_live_store: true
 
 metrics_generator:
   registry:
@@ -39,19 +61,34 @@ metrics_generator:
     remote_write:
       - url: http://prometheus:9090/api/v1/write
         send_exemplars: true
-  traces_storage:
-    path: /var/tempo/generator/traces
+
+query_frontend:
+  rf1_after: "1999-01-01T00:00:00Z"
+  mcp_server:
+    enabled: true
 
 storage:
   trace:
-    backend: local # backend configuration to use
+    backend: local
     wal:
-      path: /var/tempo/wal # where to store the wal locally
+      path: /var/tempo/wal
     local:
       path: /var/tempo/blocks
 
 overrides:
   defaults:
     metrics_generator:
-      processors: [ service-graphs, span-metrics, local-blocks ] # enables metrics generator
-      generate_native_histograms: both
+      processors: ["span-metrics", "service-graphs", "local-blocks"]
+      generate_native_histograms: both
+
+ingest:
+  enabled: true
+  kafka:
+    address: redpanda:9092
+    topic: tempo-ingest
+
+block_builder:
+  consume_cycle_duration: 30s
+
+usage_report:
+  reporting_enabled: false

.docker/openobserve-otel/README.md

@@ -5,71 +5,57 @@
 
 English | [中文](README_ZH.md)
 
-This directory contains the configuration files for setting up an observability stack with OpenObserve and OpenTelemetry
-Collector.
+This directory contains the configuration for an **alternative** observability stack using OpenObserve.
 
-### Overview
+## ⚠️ Note
 
-This setup provides a complete observability solution for your applications:
+For the **recommended** observability stack (Prometheus, Grafana, Tempo, Loki), please see `../observability/`.
 
-- **OpenObserve**: A modern, open-source observability platform for logs, metrics, and traces.
-- **OpenTelemetry Collector**: Collects and processes telemetry data before sending it to OpenObserve.
+## 🌟 Overview
 
-### Setup Instructions
+OpenObserve is a lightweight, all-in-one observability platform that handles logs, metrics, and traces in a single binary. This setup is ideal for:
+- Resource-constrained environments.
+- Quick setup and testing.
+- Users who prefer a unified UI.
 
-1. **Prerequisites**:
-    - Docker and Docker Compose installed
-    - Sufficient memory resources (minimum 2GB recommended)
+## 🚀 Quick Start
 
-2. **Starting the Services**:
-   ```bash
-   cd .docker/openobserve-otel
-   docker compose -f docker-compose.yml  up -d
-   ```
+### 1. Start Services
 
-3. **Accessing the Dashboard**:
-    - OpenObserve UI: http://localhost:5080
-    - Default credentials:
-        - Username: root@rustfs.com
-        - Password: rustfs123
+```bash
+cd .docker/openobserve-otel
+docker compose up -d
+```
 
-### Configuration
+### 2. Access Dashboard
 
-#### OpenObserve Configuration
+- **URL**: [http://localhost:5080](http://localhost:5080)
+- **Username**: `root@rustfs.com`
+- **Password**: `rustfs123`
 
-The OpenObserve service is configured with:
+## 🛠️ Configuration
 
-- Root user credentials
-- Data persistence through a volume mount
-- Memory cache enabled
-- Health checks
-- Exposed ports:
-    - 5080: HTTP API and UI
-    - 5081: OTLP gRPC
+### OpenObserve
 
-#### OpenTelemetry Collector Configuration
+- **Persistence**: Data is persisted to a Docker volume.
+- **Ports**:
+    - `5080`: HTTP API and UI
+    - `5081`: OTLP gRPC
 
-The collector is configured to:
+### OpenTelemetry Collector
 
-- Receive telemetry data via OTLP (HTTP and gRPC)
-- Collect logs from files
-- Process data in batches
-- Export data to OpenObserve
-- Manage memory usage
+- **Receivers**: OTLP (gRPC `4317`, HTTP `4318`)
+- **Exporters**: Sends data to OpenObserve.
 
-### Integration with Your Application
+## 🔗 Integration
 
-To send telemetry data from your application, configure your OpenTelemetry SDK to send data to:
+Configure your application to send OTLP data to the collector:
 
-- OTLP gRPC: `localhost:4317`
-- OTLP HTTP: `localhost:4318`
+- **Endpoint**: `http://localhost:4318` (HTTP) or `localhost:4317` (gRPC)
 
-For example, in a Rust application using the `rustfs-obs` library:
+Example for RustFS:
 
 ```bash
 export RUSTFS_OBS_ENDPOINT=http://localhost:4318
-export RUSTFS_OBS_SERVICE_NAME=yourservice
-export RUSTFS_OBS_SERVICE_VERSION=1.0.0
-export RUSTFS_OBS_ENVIRONMENT=development
+export RUSTFS_OBS_SERVICE_NAME=rustfs-node-1
 ```
-

.docker/openobserve-otel/README_ZH.md

@@ -5,71 +5,57 @@
 
 [English](README.md) | 中文
 
-## 中文
+本目录包含使用 OpenObserve 的**替代**可观测性技术栈配置。
 
-本目录包含搭建 OpenObserve 和 OpenTelemetry Collector 可观测性栈的配置文件。
+## ⚠️ 注意
 
-### 概述
+对于**推荐**的可观测性技术栈（Prometheus, Grafana, Tempo, Loki），请参阅 `../observability/`。
 
-此设置为应用程序提供了完整的可观测性解决方案：
+## 🌟 概览
 
-- **OpenObserve**：现代化、开源的可观测性平台，用于日志、指标和追踪。
-- **OpenTelemetry Collector**：收集和处理遥测数据，然后将其发送到 OpenObserve。
+OpenObserve 是一个轻量级、一体化的可观测性平台，在一个二进制文件中处理日志、指标和追踪。此设置非常适合：
+- 资源受限的环境。
+- 快速设置和测试。
+- 喜欢统一 UI 的用户。
 
-### 设置说明
+## 🚀 快速开始
 
-1. **前提条件**：
-    - 已安装 Docker 和 Docker Compose
-    - 足够的内存资源（建议至少 2GB）
-
-2. **启动服务**：
-   ```bash
-   cd .docker/openobserve-otel
-   docker compose -f docker-compose.yml  up -d
-   ```
-
-3. **访问仪表板**：
-    - OpenObserve UI：http://localhost:5080
-    - 默认凭据：
-        - 用户名：root@rustfs.com
-        - 密码：rustfs123
-
-### 配置
-
-#### OpenObserve 配置
-
-OpenObserve 服务配置：
-
-- 根用户凭据
-- 通过卷挂载实现数据持久化
-- 启用内存缓存
-- 健康检查
-- 暴露端口：
-    - 5080：HTTP API 和 UI
-    - 5081：OTLP gRPC
-
-#### OpenTelemetry Collector 配置
-
-收集器配置为：
-
-- 通过 OTLP（HTTP 和 gRPC）接收遥测数据
-- 从文件中收集日志
-- 批处理数据
-- 将数据导出到 OpenObserve
-- 管理内存使用
-
-### 与应用程序集成
-
-要从应用程序发送遥测数据，将 OpenTelemetry SDK 配置为发送数据到：
-
-- OTLP gRPC:`localhost:4317`
-- OTLP HTTP:`localhost:4318`
-
-例如，在使用 `rustfs-obs` 库的 Rust 应用程序中：
+### 1. 启动服务
 
 ```bash
-export RUSTFS_OBS_ENDPOINT=http://localhost:4317
-export RUSTFS_OBS_SERVICE_NAME=yourservice
-export RUSTFS_OBS_SERVICE_VERSION=1.0.0
-export RUSTFS_OBS_ENVIRONMENT=development
-```
+cd .docker/openobserve-otel
+docker compose up -d
+```
+
+### 2. 访问仪表盘
+
+- **URL**: [http://localhost:5080](http://localhost:5080)
+- **用户名**: `root@rustfs.com`
+- **密码**: `rustfs123`
+
+## 🛠️ 配置
+
+### OpenObserve
+
+- **持久化**: 数据持久化到 Docker 卷。
+- **端口**:
+    - `5080`: HTTP API 和 UI
+    - `5081`: OTLP gRPC
+
+### OpenTelemetry Collector
+
+- **接收器**: OTLP (gRPC `4317`, HTTP `4318`)
+- **导出器**: 将数据发送到 OpenObserve。
+
+## 🔗 集成
+
+配置您的应用程序将 OTLP 数据发送到收集器：
+
+- **端点**: `http://localhost:4318` (HTTP) 或 `localhost:4317` (gRPC)
+
+RustFS 示例：
+
+```bash
+export RUSTFS_OBS_ENDPOINT=http://localhost:4318
+export RUSTFS_OBS_SERVICE_NAME=rustfs-node-1
+```

.github/dependabot.yml

@@ -29,6 +29,8 @@ updates:
     ignore:
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
+      - dependency-name: "libunftp"
+        versions: [ "0.23.x" ]
     groups:
       s3s:
         update-types:

.github/workflows/build.yml

@@ -161,42 +161,52 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     env:
-      RUSTFLAGS: ${{ matrix.cross == 'false' && '-C target-cpu=native' || '' }}
+      RUSTFLAGS: ${{ matrix.rustflags }}
     strategy:
       fail-fast: false
       matrix:
         include:
           # Linux builds
+          # Use x86-64-v2 (SSE4.2 baseline) instead of native to ensure distributed
+          # binaries run on older x86_64 CPUs (e.g. Intel Celeron/Atom, Synology NAS).
+          # See: https://github.com/rustfs/rustfs/issues/1838
           - os: ubicloud-standard-2
             target: x86_64-unknown-linux-musl
             cross: false
             platform: linux
+            rustflags: ''
           - os: ubicloud-standard-2
             target: aarch64-unknown-linux-musl
             cross: true
             platform: linux
+            rustflags: ''
           - os: ubicloud-standard-2
             target: x86_64-unknown-linux-gnu
             cross: false
             platform: linux
+            rustflags: ''
           - os: ubicloud-standard-2
             target: aarch64-unknown-linux-gnu
             cross: true
             platform: linux
+            rustflags: ''
           # macOS builds
           - os: macos-latest
             target: aarch64-apple-darwin
             cross: false
             platform: macos
+            rustflags: ''
           - os: macos-latest
             target: x86_64-apple-darwin
             cross: false
             platform: macos
+            rustflags: ''
           # Windows builds (temporarily disabled)
           - os: windows-latest
             target: x86_64-pc-windows-msvc
             cross: false
             platform: windows
+            rustflags: ''
           #- os: windows-latest
           #  target: aarch64-pc-windows-msvc
           #  cross: true

.github/workflows/ci.yml

@@ -19,7 +19,6 @@ on:
     branches: [ main ]
     paths-ignore:
       - "**.md"
-      - "**.txt"
       - "docs/**"
       - "deploy/**"
       - "scripts/dev_*.sh"
@@ -39,7 +38,6 @@ on:
     branches: [ main ]
     paths-ignore:
       - "**.md"
-      - "**.txt"
       - "docs/**"
       - "deploy/**"
       - "scripts/dev_*.sh"
@@ -132,6 +130,9 @@ jobs:
       - name: Run clippy lints
         run: cargo clippy --all-targets --all-features -- -D warnings
 
+      - name: Check layered dependencies
+        run: ./scripts/check_layer_dependencies.sh
+
   e2e-tests:
     name: End-to-End Tests
     needs: skip-check
@@ -180,3 +181,43 @@ jobs:
           name: e2e-test-logs-${{ github.run_number }}
           path: /tmp/rustfs.log
           retention-days: 3
+
+  s3-implemented-tests:
+    name: S3 Implemented Tests
+    needs: skip-check
+    if: needs.skip-check.outputs.should_skip != 'true'
+    runs-on: ubicloud-standard-4
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-s3tests-${{ hashFiles('**/Cargo.lock') }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build debug binary
+        run: |
+          touch rustfs/build.rs
+          cargo build -p rustfs --bins --jobs 2
+
+      - name: Run implemented s3-tests
+        run: |
+          DEPLOY_MODE=binary \
+          RUSTFS_BINARY=./target/debug/rustfs \
+          TEST_MODE=single \
+          MAXFAIL=1 \
+          ./scripts/s3-tests/run.sh
+
+      - name: Upload s3 test artifacts
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: s3tests-implemented-${{ github.run_number }}
+          path: artifacts/s3tests-single/**
+          if-no-files-found: ignore
+          retention-days: 3

.github/workflows/nix-flake-update.yml

@@ -0,0 +1,61 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Update Nix Flake
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 0' # Weekly on Sundays
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  update-flake:
+    name: Update flake.lock
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Install Nix
+        uses: DeterminateSystems/determinate-nix-action@v3
+
+      - name: Check Nix flake inputs
+        uses: DeterminateSystems/flake-checker-action@v12
+
+      - name: Update flake.lock
+        id: update
+        uses: DeterminateSystems/update-flake-lock@main
+        with:
+          pr-title: "chore(deps): update flake.lock"
+          pr-labels: |
+            dependencies
+            nix
+            automated
+          commit-msg: "chore(deps): update flake.lock"
+          pr-reviewers: houseme, heihutu
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Log PR details
+        if: steps.update.outputs.pull-request-number
+        run: |
+          echo "Pull Request created: ${{ steps.update.outputs.pull-request-number }}"

.github/workflows/nix.yml

@@ -0,0 +1,84 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Nix CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'flake.nix'
+      - 'flake.lock'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
+      - '.github/workflows/nix.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'flake.nix'
+      - 'flake.lock'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
+      - '.github/workflows/nix.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+permissions:
+  contents: read
+
+jobs:
+  nix-validation:
+    name: Nix Build & Check
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: Setup Magic Nix Cache
+        uses: DeterminateSystems/magic-nix-cache-action@v13
+        
+      - name: Setup Flake Checker
+        uses: DeterminateSystems/flake-checker-action@v12
+
+      - name: Verify Flake
+        run: |
+          echo "Checking flake structure and evaluation..."
+          nix flake show
+          nix flake check --print-build-logs
+
+      - name: Build RustFS
+        run: |
+          echo "Building the default package..."
+          nix build .#default --print-build-logs
+
+      - name: Test Binary
+        run: |
+          echo "Verifying the built binary..."
+          if [ -x "./result/bin/rustfs" ]; then
+              ./result/bin/rustfs --help
+              echo "Binary verification successful."
+          else
+              echo "Error: Binary not found or not executable at ./result/bin/rustfs"
+              exit 1
+          fi

.gitignore

@@ -2,6 +2,7 @@
 .DS_Store
 .idea
 .vscode
+.cursor
 .direnv/
 /test
 /logs
@@ -39,4 +40,8 @@ artifacts/
 PR_DESCRIPTION.md
 IMPLEMENTATION_PLAN.md
 scripts/s3-tests/selected_tests.txt
-docs
+docs
+
+# nix stuff
+result*
+*.gz

.rules.md

@@ -1,702 +0,0 @@
-# RustFS Project AI Coding Rules
-
-## 🚨🚨🚨 CRITICAL DEVELOPMENT RULES - ZERO TOLERANCE 🚨🚨🚨
-
-### ⛔️ ABSOLUTE PROHIBITION: NEVER COMMIT DIRECTLY TO MASTER/MAIN BRANCH ⛔️
-
-**🔥 THIS IS THE MOST CRITICAL RULE - VIOLATION WILL RESULT IN IMMEDIATE REVERSAL 🔥**
-
-- **🚫 ZERO DIRECT COMMITS TO MAIN/MASTER BRANCH - ABSOLUTELY FORBIDDEN**
-- **🚫 ANY DIRECT COMMIT TO MAIN BRANCH MUST BE IMMEDIATELY REVERTED**
-- **🚫 NO EXCEPTIONS FOR HOTFIXES, EMERGENCIES, OR URGENT CHANGES**
-- **🚫 NO EXCEPTIONS FOR SMALL CHANGES, TYPOS, OR DOCUMENTATION UPDATES**
-- **🚫 NO EXCEPTIONS FOR ANYONE - MAINTAINERS, CONTRIBUTORS, OR ADMINS**
-
-### 📋 MANDATORY WORKFLOW - STRICTLY ENFORCED
-
-**EVERY SINGLE CHANGE MUST FOLLOW THIS WORKFLOW:**
-
-1. **Check current branch**: `git branch` (MUST NOT be on main/master)
-2. **Switch to main**: `git checkout main`
-3. **Pull latest**: `git pull origin main`
-4. **Create feature branch**: `git checkout -b feat/your-feature-name`
-5. **Make changes ONLY on feature branch**
-6. **Test thoroughly before committing**
-7. **Commit and push to feature branch**: `git push origin feat/your-feature-name`
-8. **Create Pull Request**: Use `gh pr create` (MANDATORY)
-9. **Wait for PR approval**: NO self-merging allowed
-10. **Merge through GitHub interface**: ONLY after approval
-
-### 🔒 ENFORCEMENT MECHANISMS
-
-- **Branch protection rules**: Main branch is protected
-- **Pre-commit hooks**: Will block direct commits to main
-- **CI/CD checks**: All PRs must pass before merging
-- **Code review requirement**: At least one approval needed
-- **Automated reversal**: Direct commits to main will be automatically reverted
-
-## 🎯 Core AI Development Principles
-
-### Five Execution Steps
-
-#### 1. Task Analysis and Planning
-- **Clear Objectives**: Deeply understand task requirements and expected results before starting coding
-- **Plan Development**: List specific files, components, and functions that need modification, explaining the reasons for changes
-- **Risk Assessment**: Evaluate the impact of changes on existing functionality, develop rollback plans
-
-#### 2. Precise Code Location
-- **File Identification**: Determine specific files and line numbers that need modification
-- **Impact Analysis**: Avoid modifying irrelevant files, clearly state the reason for each file modification
-- **Minimization Principle**: Unless explicitly required by the task, do not create new abstraction layers or refactor existing code
-
-#### 3. Minimal Code Changes
-- **Focus on Core**: Only write code directly required by the task
-- **Avoid Redundancy**: Do not add unnecessary logs, comments, tests, or error handling
-- **Isolation**: Ensure new code does not interfere with existing functionality, maintain code independence
-
-#### 4. Strict Code Review
-- **Correctness Check**: Verify the correctness and completeness of code logic
-- **Style Consistency**: Ensure code conforms to established project coding style
-- **Side Effect Assessment**: Evaluate the impact of changes on downstream systems
-
-#### 5. Clear Delivery Documentation
-- **Change Summary**: Detailed explanation of all modifications and reasons
-- **File List**: List all modified files and their specific changes
-- **Risk Statement**: Mark any assumptions or potential risk points
-
-### Core Principles
-- **🎯 Precise Execution**: Strictly follow task requirements, no arbitrary innovation
-- **⚡ Efficient Development**: Avoid over-design, only do necessary work
-- **🛡️ Safe and Reliable**: Always follow development processes, ensure code quality and system stability
-- **🔒 Cautious Modification**: Only modify when clearly knowing what needs to be changed and having confidence
-
-### Additional AI Behavior Rules
-
-1. **Use English for all code comments and documentation** - All comments, variable names, function names, documentation, and user-facing text in code should be in English
-2. **Clean up temporary scripts after use** - Any temporary scripts, test files, or helper files created during AI work should be removed after task completion
-3. **Only make confident modifications** - Do not make speculative changes or "convenient" modifications outside the task scope. If uncertain about a change, ask for clarification rather than guessing
-
-## Project Overview
-
-RustFS is a high-performance distributed object storage system written in Rust, compatible with S3 API. The project adopts a modular architecture, supporting erasure coding storage, multi-tenant management, observability, and other enterprise-level features.
-
-## Core Architecture Principles
-
-### 1. Modular Design
-
-- Project uses Cargo workspace structure, containing multiple independent crates
-- Core modules: `rustfs` (main service), `ecstore` (erasure coding storage), `common` (shared components)
-- Functional modules: `iam` (identity management), `madmin` (management interface), `crypto` (encryption), etc.
-- Tool modules: `cli` (command line tool), `crates/*` (utility libraries)
-
-### 2. Asynchronous Programming Pattern
-
-- Comprehensive use of `tokio` async runtime
-- Prioritize `async/await` syntax
-- Use `async-trait` for async methods in traits
-- Avoid blocking operations, use `spawn_blocking` when necessary
-
-### 3. Error Handling Strategy
-
-- **Use modular, type-safe error handling with `thiserror`**
-- Each module should define its own error type using `thiserror::Error` derive macro
-- Support error chains and context information through `#[from]` and `#[source]` attributes
-- Use `Result<T>` type aliases for consistency within each module
-- Error conversion between modules should use explicit `From` implementations
-- Follow the pattern: `pub type Result<T> = core::result::Result<T, Error>`
-- Use `#[error("description")]` attributes for clear error messages
-- Support error downcasting when needed through `other()` helper methods
-- Implement `Clone` for errors when required by the domain logic
-
-## Code Style Guidelines
-
-### 1. Formatting Configuration
-
-```toml
-max_width = 130
-fn_call_width = 90
-single_line_let_else_max_width = 100
-```
-
-### 2. **🔧 MANDATORY Code Formatting Rules**
-
-**CRITICAL**: All code must be properly formatted before committing. This project enforces strict formatting standards to maintain code consistency and readability.
-
-#### Pre-commit Requirements (MANDATORY)
-
-Before every commit, you **MUST**:
-
-1. **Format your code**:
-   ```bash
-   cargo fmt --all
-   ```
-
-2. **Verify formatting**:
-   ```bash
-   cargo fmt --all --check
-   ```
-
-3. **Pass clippy checks**:
-   ```bash
-   cargo clippy --all-targets --all-features -- -D warnings
-   ```
-
-4. **Ensure compilation**:
-   ```bash
-   cargo check --all-targets
-   ```
-
-#### Quick Commands
-
-Use these convenient Makefile targets for common tasks:
-
-```bash
-# Format all code
-make fmt
-
-# Check if code is properly formatted
-make fmt-check
-
-# Run clippy checks
-make clippy
-
-# Run compilation check
-make check
-
-# Run tests
-make test
-
-# Run all pre-commit checks (format + clippy + check + test)
-make pre-commit
-
-# Setup git hooks (one-time setup)
-make setup-hooks
-```
-
-### 3. Naming Conventions
-
-- Use `snake_case` for functions, variables, modules
-- Use `PascalCase` for types, traits, enums
-- Constants use `SCREAMING_SNAKE_CASE`
-- Global variables prefix `GLOBAL_`, e.g., `GLOBAL_Endpoints`
-- Use meaningful and descriptive names for variables, functions, and methods
-- Avoid meaningless names like `temp`, `data`, `foo`, `bar`, `test123`
-- Choose names that clearly express the purpose and intent
-
-### 4. Type Declaration Guidelines
-
-- **Prefer type inference over explicit type declarations** when the type is obvious from context
-- Let the Rust compiler infer types whenever possible to reduce verbosity and improve maintainability
-- Only specify types explicitly when:
-  - The type cannot be inferred by the compiler
-  - Explicit typing improves code clarity and readability
-  - Required for API boundaries (function signatures, public struct fields)
-  - Needed to resolve ambiguity between multiple possible types
-
-### 5. Documentation Comments
-
-- Public APIs must have documentation comments
-- Use `///` for documentation comments
-- Complex functions add `# Examples` and `# Parameters` descriptions
-- Error cases use `# Errors` descriptions
-- Always use English for all comments and documentation
-- Avoid meaningless comments like "debug 111" or placeholder text
-
-### 6. Import Guidelines
-
-- Standard library imports first
-- Third-party crate imports in the middle
-- Project internal imports last
-- Group `use` statements with blank lines between groups
-
-## Asynchronous Programming Guidelines
-
-### 1. Trait Definition
-
-```rust
-#[async_trait::async_trait]
-pub trait StorageAPI: Send + Sync {
-    async fn get_object(&self, bucket: &str, object: &str) -> Result<ObjectInfo>;
-}
-```
-
-### 2. Error Handling
-
-```rust
-// Use ? operator to propagate errors
-async fn example_function() -> Result<()> {
-    let data = read_file("path").await?;
-    process_data(data).await?;
-    Ok(())
-}
-```
-
-### 3. Concurrency Control
-
-- Use `Arc` and `Mutex`/`RwLock` for shared state management
-- Prioritize async locks from `tokio::sync`
-- Avoid holding locks for long periods
-
-## Logging and Tracing Guidelines
-
-### 1. Tracing Usage
-
-```rust
-#[tracing::instrument(skip(self, data))]
-async fn process_data(&self, data: &[u8]) -> Result<()> {
-    info!("Processing {} bytes", data.len());
-    // Implementation logic
-}
-```
-
-### 2. Log Levels
-
-- `error!`: System errors requiring immediate attention
-- `warn!`: Warning information that may affect functionality
-- `info!`: Important business information
-- `debug!`: Debug information for development use
-- `trace!`: Detailed execution paths
-
-### 3. Structured Logging
-
-```rust
-info!(
-    counter.rustfs_api_requests_total = 1_u64,
-    key_request_method = %request.method(),
-    key_request_uri_path = %request.uri().path(),
-    "API request processed"
-);
-```
-
-## Error Handling Guidelines
-
-### 1. Error Type Definition
-
-```rust
-// Use thiserror for module-specific error types
-#[derive(thiserror::Error, Debug)]
-pub enum MyError {
-    #[error("IO error: {0}")]
-    Io(#[from] std::io::Error),
-
-    #[error("Storage error: {0}")]
-    Storage(#[from] ecstore::error::StorageError),
-
-    #[error("Custom error: {message}")]
-    Custom { message: String },
-
-    #[error("File not found: {path}")]
-    FileNotFound { path: String },
-
-    #[error("Invalid configuration: {0}")]
-    InvalidConfig(String),
-}
-
-// Provide Result type alias for the module
-pub type Result<T> = core::result::Result<T, MyError>;
-```
-
-### 2. Error Helper Methods
-
-```rust
-impl MyError {
-    /// Create error from any compatible error type
-    pub fn other<E>(error: E) -> Self
-    where
-        E: Into<Box<dyn std::error::Error + Send + Sync>>,
-    {
-        MyError::Io(std::io::Error::other(error))
-    }
-}
-```
-
-### 3. Error Context and Propagation
-
-```rust
-// Use ? operator for clean error propagation
-async fn example_function() -> Result<()> {
-    let data = read_file("path").await?;
-    process_data(data).await?;
-    Ok(())
-}
-
-// Add context to errors
-fn process_with_context(path: &str) -> Result<()> {
-    std::fs::read(path)
-        .map_err(|e| MyError::Custom {
-            message: format!("Failed to read {}: {}", path, e)
-        })?;
-    Ok(())
-}
-```
-
-## Performance Optimization Guidelines
-
-### 1. Memory Management
-
-- Use `Bytes` instead of `Vec<u8>` for zero-copy operations
-- Avoid unnecessary cloning, use reference passing
-- Use `Arc` for sharing large objects
-
-### 2. Concurrency Optimization
-
-```rust
-// Use join_all for concurrent operations
-let futures = disks.iter().map(|disk| disk.operation());
-let results = join_all(futures).await;
-```
-
-### 3. Caching Strategy
-
-- Use `LazyLock` for global caching
-- Implement LRU cache to avoid memory leaks
-
-## Testing Guidelines
-
-### 1. Unit Tests
-
-```rust
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use test_case::test_case;
-
-    #[tokio::test]
-    async fn test_async_function() {
-        let result = async_function().await;
-        assert!(result.is_ok());
-    }
-
-    #[test_case("input1", "expected1")]
-    #[test_case("input2", "expected2")]
-    fn test_with_cases(input: &str, expected: &str) {
-        assert_eq!(function(input), expected);
-    }
-}
-```
-
-### 2. Integration Tests
-
-- Use `e2e_test` module for end-to-end testing
-- Simulate real storage environments
-
-### 3. Test Quality Standards
-
-- Write meaningful test cases that verify actual functionality
-- Avoid placeholder or debug content like "debug 111", "test test", etc.
-- Use descriptive test names that clearly indicate what is being tested
-- Each test should have a clear purpose and verify specific behavior
-- Test data should be realistic and representative of actual use cases
-
-## Cross-Platform Compatibility Guidelines
-
-### 1. CPU Architecture Compatibility
-
-- **Always consider multi-platform and different CPU architecture compatibility** when writing code
-- Support major architectures: x86_64, aarch64 (ARM64), and other target platforms
-- Use conditional compilation for architecture-specific code:
-
-```rust
-#[cfg(target_arch = "x86_64")]
-fn optimized_x86_64_function() { /* x86_64 specific implementation */ }
-
-#[cfg(target_arch = "aarch64")]
-fn optimized_aarch64_function() { /* ARM64 specific implementation */ }
-
-#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
-fn generic_function() { /* Generic fallback implementation */ }
-```
-
-### 2. Platform-Specific Dependencies
-
-- Use feature flags for platform-specific dependencies
-- Provide fallback implementations for unsupported platforms
-- Test on multiple architectures in CI/CD pipeline
-
-### 3. Endianness Considerations
-
-- Use explicit byte order conversion when dealing with binary data
-- Prefer `to_le_bytes()`, `from_le_bytes()` for consistent little-endian format
-- Use `byteorder` crate for complex binary format handling
-
-### 4. SIMD and Performance Optimizations
-
-- Use portable SIMD libraries like `wide` or `packed_simd`
-- Provide fallback implementations for non-SIMD architectures
-- Use runtime feature detection when appropriate
-
-## Security Guidelines
-
-### 1. Memory Safety
-
-- Disable `unsafe` code (workspace.lints.rust.unsafe_code = "deny")
-- Use `rustls` instead of `openssl`
-
-### 2. Authentication and Authorization
-
-```rust
-// Use IAM system for permission checks
-let identity = iam.authenticate(&access_key, &secret_key).await?;
-iam.authorize(&identity, &action, &resource).await?;
-```
-
-## Configuration Management Guidelines
-
-### 1. Environment Variables
-
-- Use `RUSTFS_` prefix
-- Support both configuration files and environment variables
-- Provide reasonable default values
-
-### 2. Configuration Structure
-
-```rust
-#[derive(Debug, Deserialize, Clone)]
-pub struct Config {
-    pub address: String,
-    pub volumes: String,
-    #[serde(default)]
-    pub console_enable: bool,
-}
-```
-
-## Dependency Management Guidelines
-
-### 1. Workspace Dependencies
-
-- Manage versions uniformly at workspace level
-- Use `workspace = true` to inherit configuration
-
-### 2. Feature Flags
-
-```rust
-[features]
-default = ["file"]
-gpu = ["dep:nvml-wrapper"]
-kafka = ["dep:rdkafka"]
-```
-
-## Deployment and Operations Guidelines
-
-### 1. Containerization
-
-- Provide Dockerfile and docker-compose configuration
-- Support multi-stage builds to optimize image size
-
-### 2. Observability
-
-- Integrate OpenTelemetry for distributed tracing
-- Support Prometheus metrics collection
-- Provide Grafana dashboards
-
-### 3. Health Checks
-
-```rust
-// Implement health check endpoint
-async fn health_check() -> Result<HealthStatus> {
-    // Check component status
-}
-```
-
-## Code Review Checklist
-
-### 1. **Code Formatting and Quality (MANDATORY)**
-
-- [ ] **Code is properly formatted** (`cargo fmt --all --check` passes)
-- [ ] **All clippy warnings are resolved** (`cargo clippy --all-targets --all-features -- -D warnings` passes)
-- [ ] **Code compiles successfully** (`cargo check --all-targets` passes)
-- [ ] **Pre-commit hooks are working** and all checks pass
-- [ ] **No formatting-related changes** mixed with functional changes (separate commits)
-
-### 2. Functionality
-
-- [ ] Are all error cases properly handled?
-- [ ] Is there appropriate logging?
-- [ ] Is there necessary test coverage?
-
-### 3. Performance
-
-- [ ] Are unnecessary memory allocations avoided?
-- [ ] Are async operations used correctly?
-- [ ] Are there potential deadlock risks?
-
-### 4. Security
-
-- [ ] Are input parameters properly validated?
-- [ ] Are there appropriate permission checks?
-- [ ] Is information leakage avoided?
-
-### 5. Cross-Platform Compatibility
-
-- [ ] Does the code work on different CPU architectures (x86_64, aarch64)?
-- [ ] Are platform-specific features properly gated with conditional compilation?
-- [ ] Is byte order handling correct for binary data?
-- [ ] Are there appropriate fallback implementations for unsupported platforms?
-
-### 6. Code Commits and Documentation
-
-- [ ] Does it comply with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)?
-- [ ] Are commit messages concise and under 72 characters for the title line?
-- [ ] Commit titles should be concise and in English, avoid Chinese
-- [ ] Is PR description provided in copyable markdown format for easy copying?
-
-## Common Patterns and Best Practices
-
-### 1. Resource Management
-
-```rust
-// Use RAII pattern for resource management
-pub struct ResourceGuard {
-    resource: Resource,
-}
-
-impl Drop for ResourceGuard {
-    fn drop(&mut self) {
-        // Clean up resources
-    }
-}
-```
-
-### 2. Dependency Injection
-
-```rust
-// Use dependency injection pattern
-pub struct Service {
-    config: Arc<Config>,
-    storage: Arc<dyn StorageAPI>,
-}
-```
-
-### 3. Graceful Shutdown
-
-```rust
-// Implement graceful shutdown
-async fn shutdown_gracefully(shutdown_rx: &mut Receiver<()>) {
-    tokio::select! {
-        _ = shutdown_rx.recv() => {
-            info!("Received shutdown signal");
-            // Perform cleanup operations
-        }
-        _ = tokio::time::sleep(SHUTDOWN_TIMEOUT) => {
-            warn!("Shutdown timeout reached");
-        }
-    }
-}
-```
-
-## Domain-Specific Guidelines
-
-### 1. Storage Operations
-
-- All storage operations must support erasure coding
-- Implement read/write quorum mechanisms
-- Support data integrity verification
-
-### 2. Network Communication
-
-- Use gRPC for internal service communication
-- HTTP/HTTPS support for S3-compatible API
-- Implement connection pooling and retry mechanisms
-
-### 3. Metadata Management
-
-- Use FlatBuffers for serialization
-- Support version control and migration
-- Implement metadata caching
-
-## Branch Management and Development Workflow
-
-### Branch Management
-
-- **🚨 CRITICAL: NEVER modify code directly on main or master branch - THIS IS ABSOLUTELY FORBIDDEN 🚨**
-- **⚠️ ANY DIRECT COMMITS TO MASTER/MAIN WILL BE REJECTED AND MUST BE REVERTED IMMEDIATELY ⚠️**
-- **🔒 ALL CHANGES MUST GO THROUGH PULL REQUESTS - NO DIRECT COMMITS TO MAIN UNDER ANY CIRCUMSTANCES 🔒**
-- **Always work on feature branches - NO EXCEPTIONS**
-- Always check the .rules.md file before starting to ensure you understand the project guidelines
-- **MANDATORY workflow for ALL changes:**
-   1. `git checkout main` (switch to main branch)
-   2. `git pull` (get latest changes)
-   3. `git checkout -b feat/your-feature-name` (create and switch to feature branch)
-   4. Make your changes ONLY on the feature branch
-   5. Test thoroughly before committing
-   6. Commit and push to the feature branch
-   7. **Create a pull request for code review - THIS IS THE ONLY WAY TO MERGE TO MAIN**
-   8. **Wait for PR approval before merging - NEVER merge your own PRs without review**
-- Use descriptive branch names following the pattern: `feat/feature-name`, `fix/issue-name`, `refactor/component-name`, etc.
-- **Double-check current branch before ANY commit: `git branch` to ensure you're NOT on main/master**
-- **Pull Request Requirements:**
-  - All changes must be submitted via PR regardless of size or urgency
-  - PRs must include comprehensive description and testing information
-  - PRs must pass all CI/CD checks before merging
-  - PRs require at least one approval from code reviewers
-  - Even hotfixes and emergency changes must go through PR process
-- **Enforcement:**
-  - Main branch should be protected with branch protection rules
-  - Direct pushes to main should be blocked by repository settings
-  - Any accidental direct commits to main must be immediately reverted via PR
-
-### Development Workflow
-
-## 🎯 **Core Development Principles**
-
-- **🔴 Every change must be precise - don't modify unless you're confident**
-  - Carefully analyze code logic and ensure complete understanding before making changes
-  - When uncertain, prefer asking users or consulting documentation over blind modifications
-  - Use small iterative steps, modify only necessary parts at a time
-  - Evaluate impact scope before changes to ensure no new issues are introduced
-
-- **🚀 GitHub PR creation prioritizes gh command usage**
-  - Prefer using `gh pr create` command to create Pull Requests
-  - Avoid having users manually create PRs through web interface
-  - Provide clear and professional PR titles and descriptions
-  - Using `gh` commands ensures better integration and automation
-
-## 📝 **Code Quality Requirements**
-
-- Use English for all code comments, documentation, and variable names
-- Write meaningful and descriptive names for variables, functions, and methods
-- Avoid meaningless test content like "debug 111" or placeholder values
-- Before each change, carefully read the existing code to ensure you understand the code structure and implementation, do not break existing logic implementation, do not introduce new issues
-- Ensure each change provides sufficient test cases to guarantee code correctness
-- Do not arbitrarily modify numbers and constants in test cases, carefully analyze their meaning to ensure test case correctness
-- When writing or modifying tests, check existing test cases to ensure they have scientific naming and rigorous logic testing, if not compliant, modify test cases to ensure scientific and rigorous testing
-- **Before committing any changes, run `cargo clippy --all-targets --all-features -- -D warnings` to ensure all code passes Clippy checks**
-- After each development completion, first git add . then git commit -m "feat: feature description" or "fix: issue description", ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
-- **Keep commit messages concise and under 72 characters** for the title line, use body for detailed explanations if needed
-- After each development completion, first git push to remote repository
-- After each change completion, summarize the changes, do not create summary files, provide a brief change description, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
-- Provide change descriptions needed for PR in the conversation, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
-- **Always provide PR descriptions in English** after completing any changes, including:
-  - Clear and concise title following Conventional Commits format
-  - Detailed description of what was changed and why
-  - List of key changes and improvements
-  - Any breaking changes or migration notes if applicable
-  - Testing information and verification steps
-- **Provide PR descriptions in copyable markdown format** enclosed in code blocks for easy one-click copying
-
-## 🚫 AI Documentation Generation Restrictions
-
-### Forbidden Summary Documents
-
-- **Strictly forbidden to create any form of AI-generated summary documents**
-- **Do not create documents containing large amounts of emoji, detailed formatting tables and typical AI style**
-- **Do not generate the following types of documents in the project:**
-  - Benchmark summary documents (BENCHMARK*.md)
-  - Implementation comparison analysis documents (IMPLEMENTATION_COMPARISON*.md)
-  - Performance analysis report documents
-  - Architecture summary documents
-  - Feature comparison documents
-  - Any documents with large amounts of emoji and formatted content
-- **If documentation is needed, only create when explicitly requested by the user, and maintain a concise and practical style**
-- **Documentation should focus on actually needed information, avoiding excessive formatting and decorative content**
-- **Any discovered AI-generated summary documents should be immediately deleted**
-
-### Allowed Documentation Types
-
-- README.md (project introduction, keep concise)
-- Technical documentation (only create when explicitly needed)
-- User manual (only create when explicitly needed)
-- API documentation (generated from code)
-- Changelog (CHANGELOG.md)
-
-These rules should serve as guiding principles when developing the RustFS project, ensuring code quality, performance, and maintainability.

AGENTS.md

@@ -18,8 +18,8 @@ Or simply run `make pre-commit` which covers all checks. **DO NOT commit if any
 
 ## Communication Rules
 
-- Respond to the user in Chinese; use English in all other contexts.
-- Code and documentation must be written in English only. Chinese text is allowed solely as test data/fixtures when a case explicitly requires Chinese-language content for validation.
+- Respond to the user in the same language used by the user.
+- Code and documentation must be written in English only.
 - **Pull Request titles and descriptions must be written in English** to ensure consistency and accessibility for all contributors.
 
 ## Project Structure
@@ -71,8 +71,10 @@ Co-locate unit tests with their modules and give behavior-led names. Integration
 
 ## Environment Variables
 
-- `RUSTFS_ENABLE_SCANNER` - Enable/disable background data scanner (default: true)
-- `RUSTFS_ENABLE_HEAL` - Enable/disable auto-heal functionality (default: true)
+- Global configuration environment variables must use flat `RUSTFS_*` names (no module segments), such as `RUSTFS_REGION`, `RUSTFS_ADDRESS`, `RUSTFS_VOLUMES`, and `RUSTFS_LICENSE`.
+- `RUSTFS_SCANNER_ENABLED` - Enable/disable background data scanner (default: true)
+- `RUSTFS_HEAL_ENABLED` - Enable/disable auto-heal functionality (default: true)
+- Deprecated aliases (for pre-beta compatibility) are documented in [the config module README](crates/config/README.md#environment-variable-naming-conventions) and must log warnings when used.
 - For KMS tests: `NO_PROXY=127.0.0.1,localhost` and clear proxy environment variables
 
 ## Commit & Pull Request Guidelines

CHANGELOG.md

@@ -0,0 +1,76 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- **OpenStack Keystone Authentication Integration**: Full support for OpenStack Keystone authentication via X-Auth-Token headers
+  - Tower-based middleware (`KeystoneAuthLayer`) self-contained within `rustfs-keystone` crate
+  - Task-local storage for async-safe credential passing between middleware and auth handlers
+  - Automatic detection of Keystone credentials (access keys prefixed with `keystone:`)
+  - Role-based permission mapping (admin/reseller_admin roles grant owner permissions)
+  - Token caching for high-performance validation with configurable cache size and TTL
+  - Dual authentication support: Keystone and standard AWS Signature v4 work simultaneously
+  - Immediate 401 response for invalid tokens (no fallback to local auth)
+  - XML-formatted error responses compatible with S3 API
+  - Comprehensive integration documentation with manual testing guide
+  - **32 unit and integration tests** covering middleware, auth handlers, task-local storage, and role detection
+
+### Changed
+- **HTTP Server Stack**: Integrated `KeystoneAuthLayer` middleware from `rustfs-keystone` crate into service stack (positioned after ReadinessGateLayer)
+- **IAMAuth**: Enhanced `get_secret_key()` to return empty secret for Keystone credentials (bypasses signature validation)
+- **Auth Module**: Modified `check_key_valid()` to retrieve Keystone credentials from task-local storage and determine admin status
+
+### Technical Details
+- Middleware is self-contained in `rustfs-keystone` crate following the trusted-proxies pattern for integration-specific middleware
+- Uses `BoxBody` pattern for Hyper 1.x compatibility
+- Task-local storage provides request-scoped credential passing without modifying HTTP request/response types
+- Integration preserves existing S3 authentication flow while adding Keystone support
+- Zero breaking changes to existing functionality
+- No new top-level directories in main binary crate (middleware lives in integration crate)
+
+### Documentation
+- Updated `crates/keystone/README.md` with complete integration architecture and workflow
+- Added detailed manual testing guide with 10 test scenarios
+- Updated main `README.md` to list Keystone authentication as available feature
+- Added troubleshooting section for common integration issues
+
+### Configuration
+New environment variables:
+- `RUSTFS_KEYSTONE_ENABLE` - Enable/disable Keystone authentication (default: false)
+- `RUSTFS_KEYSTONE_AUTH_URL` - Keystone API endpoint URL
+- `RUSTFS_KEYSTONE_VERSION` - Keystone API version (v3)
+- `RUSTFS_KEYSTONE_ADMIN_USER` - Admin username for privileged operations
+- `RUSTFS_KEYSTONE_ADMIN_PASSWORD` - Admin password
+- `RUSTFS_KEYSTONE_ADMIN_PROJECT` - Admin project name
+- `RUSTFS_KEYSTONE_ADMIN_DOMAIN` - Admin domain name (default: Default)
+- `RUSTFS_KEYSTONE_CACHE_SIZE` - Token cache size (default: 10000)
+- `RUSTFS_KEYSTONE_CACHE_TTL` - Token cache TTL in seconds (default: 300)
+- `RUSTFS_KEYSTONE_VERIFY_SSL` - Verify SSL certificates (default: true)
+
+### Files Modified
+- `crates/keystone/src/middleware.rs` - Created Keystone authentication middleware (self-contained in keystone crate)
+- `crates/keystone/src/lib.rs` - Exported middleware module and KEYSTONE_CREDENTIALS
+- `crates/keystone/Cargo.toml` - Added Tower/HTTP dependencies for middleware functionality
+- `rustfs/src/server/http.rs` - Integrated KeystoneAuthLayer from rustfs-keystone crate
+- `rustfs/src/auth.rs` - Enhanced IAMAuth and check_key_valid for Keystone support, imported KEYSTONE_CREDENTIALS from rustfs-keystone
+- `crates/keystone/README.md` - Comprehensive integration documentation
+- `README.md` - Added Keystone as available feature
+
+### Testing
+- 16 unit tests in rustfs-keystone crate (config, auth, middleware, identity)
+- 10 integration tests in rustfs-keystone crate (task-local storage, middleware layer, scope isolation)
+- 6 auth unit tests in rustfs crate (role detection, task-local storage, Keystone credential handling)
+- **Total: 32 tests** passing with zero compilation errors
+- Manual testing guide provided for end-to-end validation
+- All tests passing with `cargo test --all --exclude e2e_test`
+
+---
+
+## Previous Releases
+
+See [GitHub Releases](https://github.com/rustfs/rustfs/releases) for previous version history.

CLA.md

@@ -1,85 +1,60 @@
-RustFS Individual Contributor License Agreement
+# RustFS Individual Contributor License Agreement
 
-Thank you for your interest in contributing documentation and related software code to a project hosted or managed by
-RustFS. In order to clarify the intellectual property license granted with Contributions from any person or entity,
-RustFS must have a Contributor License Agreement ("CLA") on file that has been signed by each Contributor, indicating
-agreement to the license terms below. This version of the Contributor License Agreement allows an individual to submit
-Contributions to the applicable project. If you are making a submission on behalf of a legal entity, then you should
-sign the separate Corporate Contributor License Agreement.
+Thank you for your interest in contributing to RustFS. In order to clarify the intellectual property license granted with Contributions from any person or entity, RustFS, Inc. ("RustFS") must have a Contributor License Agreement ("CLA") on file that has been signed by each Contributor, indicating agreement to the license terms below. This license is for your protection as a Contributor as well as the protection of RustFS and its users; it does not change your rights to use your own Contributions for any other purpose.
 
-You accept and agree to the following terms and conditions for Your present and future Contributions submitted to
-RustFS. You hereby irrevocably assign and transfer to RustFS all right, title, and interest in and to Your
-Contributions, including all copyrights and other intellectual property rights therein.
+You accept and agree to the following terms and conditions for Your present and future Contributions submitted to RustFS. Except for the license granted herein to RustFS and recipients of software distributed by RustFS, You reserve all right, title, and interest in and to Your Contributions.
 
-Definitions
+**You understand and agree that You will not receive any royalty or other compensation for Your Contributions.**
 
-“You” (or “Your”) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this
-Agreement with RustFS. For legal entities, the entity making a Contribution and all other entities that control, are
-controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes
-of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such
-entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares,
-or (iii) beneficial ownership of such entity.
+## 1. Definitions
 
-“Contribution” shall mean any original work of authorship, including any modifications or additions to an existing work,
-that is intentionally submitted by You to RustFS for inclusion in, or documentation of, any of the products or projects
-owned or managed by RustFS (the "Work"), including without limitation any Work described in Schedule A. For the purposes
-of this definition, "submitted" means any form of electronic or written communication sent to RustFS or its
-representatives, including but not limited to communication on electronic mailing lists, source code control systems,
-and issue tracking systems that are managed by, or on behalf of, RustFS for the purpose of discussing and improving the
-Work.
+*   **"You"** (or **"Your"**) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with RustFS. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
 
-Assignment of Copyright
+*   **"Contribution"** shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to RustFS for inclusion in, or documentation of, any of the products or projects owned or managed by RustFS (the "Work"). For the purposes of this definition, "submitted" means any form of electronic or written communication sent to RustFS or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, RustFS for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution."
 
-Subject to the terms and conditions of this Agreement, You hereby irrevocably assign and transfer to RustFS all right,
-title, and interest in and to Your Contributions, including all copyrights and other intellectual property rights
-therein, for the entire term of such rights, including all renewals and extensions. You agree to execute all documents
-and take all actions as may be reasonably necessary to vest in RustFS the ownership of Your Contributions and to assist
-RustFS in perfecting, maintaining, and enforcing its rights in Your Contributions.
+## 2. Grant of Copyright License
 
-Grant of Patent License
+Subject to the terms and conditions of this Agreement, You hereby grant to RustFS and to recipients of software distributed by RustFS a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works under any license, including proprietary or commercial licenses and open-source licenses, at RustFS's sole discretion.
 
-Subject to the terms and conditions of this Agreement, You hereby grant to RustFS and to recipients of documentation and
-software distributed by RustFS a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as
-stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the
-Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your
-Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was
-submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or
-counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes
-direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for
-that Contribution or Work shall terminate as of the date such litigation is filed.
+## 3. Grant of Patent License
 
-You represent that you are legally entitled to grant the above assignment and license.
+Subject to the terms and conditions of this Agreement, You hereby grant to RustFS and to recipients of software distributed by RustFS a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.
 
-You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of
-others). You represent that Your Contribution submissions include complete details of any third-party license or other
-restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which
-are associated with any part of Your Contributions.
+## 4. Representations
 
-You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You
-may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You
-provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
-including, without limitation, any warranties or conditions of TITLE, NON- INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR
-A PARTICULAR PURPOSE.
+You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to RustFS, or that your employer has executed a separate Corporate CLA with RustFS.
 
-Should You wish to submit work that is not Your original creation, You may submit it to RustFS separately from any
-Contribution, identifying the complete details of its source and of any license or other restriction (including, but not
-limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously
-marking the work as "Submitted on behalf of a third-party: [named here]”.
+You represent that each of Your Contributions is Your original creation. You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which are associated with any part of Your Contributions.
 
-You agree to notify RustFS of any facts or circumstances of which you become aware that would make these representations
-inaccurate in any respect.
+## 5. Support and Warranty
 
-Modification of CLA
+You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of **TITLE**, **NON-INFRINGEMENT**, **MERCHANTABILITY**, or **FITNESS FOR A PARTICULAR PURPOSE**.
 
-RustFS reserves the right to update or modify this CLA in the future. Any updates or modifications to this CLA shall
-apply only to Contributions made after the effective date of the revised CLA. Contributions made prior to the update
-shall remain governed by the version of the CLA that was in effect at the time of submission. It is not necessary for
-all Contributors to re-sign the CLA when the CLA is updated or modified.
+## 6. Third-Party Work
 
-Governing Law and Dispute Resolution
+Should You wish to submit work that is not Your original creation, You may submit it to RustFS separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as "Submitted on behalf of a third-party: [named here]".
 
-This Agreement will be governed by and construed in accordance with the laws of the People's Republic of China excluding
-that body of laws known as conflict of laws. The parties expressly agree that the United Nations Convention on Contracts
-for the International Sale of Goods will not apply. Any legal action or proceeding arising under this Agreement will be
-brought exclusively in the courts located in Beijing, China, and the parties hereby irrevocably consent to the personal
-jurisdiction and venue therein.
+## 7. Governing Law and Jurisdiction
+
+This Agreement shall be governed by and construed in accordance with the laws of the State of Delaware, United States of America, without regard to its conflict of laws principles. The parties expressly agree that the United Nations Convention on Contracts for the International Sale of Goods will not apply. Any legal action or proceeding arising under or in connection with this Agreement shall be brought exclusively in the state or federal courts located in the State of Delaware, United States of America, and the parties hereby irrevocably consent to the personal jurisdiction and venue therein.
+
+## 8. Severability
+
+If any provision of this Agreement is found to be invalid or unenforceable, the remaining provisions will continue in full force and effect.
+
+## 9. Entire Agreement and Assignment
+
+This Agreement constitutes the entire agreement between the parties with respect to the subject matter hereof and supersedes all prior agreements, understandings, negotiations, and discussions, whether written or oral. RustFS may assign its rights and obligations under this Agreement to any third party. You may not assign Your rights or obligations under this Agreement without the prior written consent of RustFS.
+
+---
+
+**Please read the terms of this Agreement carefully. By submitting a Contribution to RustFS, You agree to be bound by the terms and conditions of this Agreement.**
+
+| | |
+|---|---|
+| **Full Name** | __________________________________ |
+| **GitHub Username** | __________________________________ |
+| **Email Address** | __________________________________ |
+| **Date** | __________________________________ |
+
+*(Electronic signature or acknowledgement via GitHub commit/Pull Request constitutes valid acceptance of this Agreement).*

Cargo.toml

@@ -27,6 +27,7 @@ members = [
     "crates/filemeta", # File metadata management
     "crates/heal", # Erasure set and object healing
     "crates/iam", # Identity and Access Management
+    "crates/keystone", # OpenStack Keystone integration
     "crates/kms", # Key Management Service
     "crates/lock", # Distributed locking implementation
     "crates/madmin", # Management dashboard and admin API interface
@@ -82,6 +83,7 @@ rustfs-crypto = { path = "crates/crypto", version = "0.0.5" }
 rustfs-ecstore = { path = "crates/ecstore", version = "0.0.5" }
 rustfs-filemeta = { path = "crates/filemeta", version = "0.0.5" }
 rustfs-iam = { path = "crates/iam", version = "0.0.5" }
+rustfs-keystone = { path = "crates/keystone", version = "0.0.5" }
 rustfs-kms = { path = "crates/kms", version = "0.0.5" }
 rustfs-lock = { path = "crates/lock", version = "0.0.5" }
 rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
@@ -105,14 +107,13 @@ rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
-async-compression = { version = "0.4.39" }
+async-compression = { version = "0.4.41" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
 axum = "0.8.8"
-axum-server = { version = "0.8.0", features = ["tls-rustls"], default-features = false }
-futures = "0.3.31"
-futures-core = "0.3.31"
-futures-util = "0.3.31"
+futures = "0.3.32"
+futures-core = "0.3.32"
+futures-util = "0.3.32"
 pollster = "0.4.0"
 hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
 hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
@@ -127,9 +128,9 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = ["logg
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
 tokio-util = { version = "0.7.18", features = ["io", "compat"] }
-tonic = { version = "0.14.3", features = ["gzip"] }
-tonic-prost = { version = "0.14.3" }
-tonic-prost-build = { version = "0.14.3" }
+tonic = { version = "0.14.5", features = ["gzip"] }
+tonic-prost = { version = "0.14.5" }
+tonic-prost-build = { version = "0.14.5" }
 tower = { version = "0.5.3", features = ["timeout"] }
 tower-http = { version = "0.6.8", features = ["cors"] }
 
@@ -140,8 +141,8 @@ byteorder = "1.5.0"
 flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.3"
-quick-xml = "0.39.0"
-rmcp = { version = "0.15.0" }
+quick-xml = "0.39.2"
+rmcp = { version = "0.17.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
@@ -157,10 +158,10 @@ chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
 hmac = { version = "0.13.0-rc.5" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
+openidconnect = { version = "4.0", default-features = false }
 pbkdf2 = "0.13.0-rc.9"
-rsa = { version = "0.10.0-rc.15" }
-rustls = { version = "0.23.36", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
-rustls-pemfile = "2.2.0"
+rsa = { version = "0.10.0-rc.16" }
+rustls = { version = "0.23.37", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types = "1.14.0"
 sha1 = "0.11.0-rc.5"
 sha2 = "0.11.0-rc.5"
@@ -168,46 +169,46 @@ subtle = "2.6"
 zeroize = { version = "1.8.2", features = ["derive"] }
 
 # Time and Date
-chrono = { version = "0.4.43", features = ["serde"] }
+chrono = { version = "0.4.44", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.19", features = ["serde"] }
+jiff = { version = "0.2.22", features = ["serde"] }
 time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
 
 # Utilities and Tools
-anyhow = "1.0.101"
-arc-swap = "1.8.1"
+anyhow = "1.0.102"
+arc-swap = "1.8.2"
 astral-tokio-tar = "0.5.6"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.13" }
-aws-credential-types = { version = "1.2.11" }
-aws-sdk-s3 = { version = "1.122.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
-aws-smithy-types = { version = "1.4.4" }
+aws-config = { version = "1.8.14" }
+aws-credential-types = { version = "1.2.13" }
+aws-sdk-s3 = { version = "1.124.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-smithy-http-client = { version = "1.1.11", default-features = false, features = ["default-client", "rustls-aws-lc"] }
+aws-smithy-types = { version = "1.4.5" }
 backtrace = "0.3.76"
 base64 = "0.22.1"
 base64-simd = "0.8.0"
 brotli = "8.0.2"
 cfg-if = "1.0.4"
-clap = { version = "4.5.57", features = ["derive", "env"] }
+clap = { version = "4.5.60", features = ["derive", "env"] }
 const-str = { version = "1.1.0", features = ["std", "proc"] }
 convert_case = "0.11.0"
 criterion = { version = "0.8", features = ["html_reports"] }
 crossbeam-queue = "0.3.12"
-datafusion = "52.1.0"
+datafusion = "52.2.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
 flate2 = "1.1.9"
-flexi_logger = { version = "0.31.8", features = ["trc", "dont_minimize_extra_stacks", "compress", "kv", "json"] }
 glob = "0.3.3"
-google-cloud-storage = "1.7.0"
-google-cloud-auth = "1.5.0"
+google-cloud-storage = "1.8.0"
+google-cloud-auth = "1.6.0"
 hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
 hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.181"
+libc = "0.2.182"
 libsystemd = "0.7.2"
 local-ip-address = "0.6.10"
 lz4 = "1.28.1"
@@ -215,27 +216,27 @@ matchit = "0.9.1"
 md-5 = "0.11.0-rc.5"
 md5 = "0.8.0"
 mime_guess = "2.0.5"
-moka = { version = "0.12.13", features = ["future"] }
+moka = { version = "0.12.14", features = ["future"] }
 netif = "0.1.6"
-nu-ansi-term = "0.50.3"
 num_cpus = { version = "1.17.0" }
-nvml-wrapper = "0.11.0"
+nvml-wrapper = "0.12.0"
 object_store = "0.12.5"
 parking_lot = "0.12.5"
 path-absolutize = "3.1.1"
 path-clean = "1.0.1"
-pin-project-lite = "0.2.16"
+pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
 rand = { version = "0.10.0", features = ["serde"] }
+ratelimit = "0.10.0"
 rayon = "1.11.0"
 reed-solomon-simd = { version = "3.1.0" }
 regex = { version = "1.12.3" }
 rumqttc = { version = "0.25.1" }
-rustix = { version = "1.1.3", features = ["fs"] }
+rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.1" }
-s3s = { version = "0.13.0-alpha.3", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", rev = "7b8e22b49bc76839e6085a4ddfa553323f1b8049" }
-serial_test = "3.3.1"
+s3s = { version = "0.13.0", features = ["minio"] }
+serial_test = "3.4.0"
 shadow-rs = { version = "1.7.0", default-features = false }
 siphasher = "1.0.2"
 smallvec = { version = "1.15.1", features = ["serde"] }
@@ -243,10 +244,10 @@ smartstring = "1.0.1"
 snafu = "0.8.9"
 snap = "1.1.1"
 starshard = { version = "1.1.0", features = ["rayon", "async", "serde"] }
-strum = { version = "0.27.2", features = ["derive"] }
-sysinfo = "0.38.1"
+strum = { version = "0.28.0", features = ["derive"] }
+sysinfo = "0.38.3"
 temp-env = "0.3.6"
-tempfile = "3.25.0"
+tempfile = "3.26.0"
 test-case = "3.3.1"
 thiserror = "2.0.18"
 tracing = { version = "0.1.44" }
@@ -257,13 +258,13 @@ tracing-subscriber = { version = "0.3.22", features = ["env-filter", "time"] }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.20.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.21.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
 vaultrs = { version = "0.7.4" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "7.4.0"
+zip = "8.1.0"
 zstd = "0.13.3"
 
 # Observability and Metrics
@@ -276,8 +277,9 @@ opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_
 opentelemetry-stdout = { version = "0.31.0" }
 
 # FTP and SFTP
-libunftp = { version = "0.22.0", features = ["experimental"] }
-suppaftp = { version = "8.0.1", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
+libunftp = { version = "0.23.0", features = ["experimental"] }
+unftp-core = "0.1.0"
+suppaftp = { version = "8.0.2", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.7"
 
 # Performance Analysis and Memory Profiling

Dockerfile

@@ -1,3 +1,17 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 FROM alpine:3.23 AS build
 
 ARG TARGETARCH

Dockerfile.glibc

@@ -1,3 +1,17 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 FROM ubuntu:24.04 AS build
 
 ARG TARGETARCH

Dockerfile.source

@@ -1,4 +1,18 @@
 # syntax=docker/dockerfile:1.6
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Multi-stage Dockerfile for RustFS - LOCAL DEVELOPMENT ONLY
 #
 # IMPORTANT: This Dockerfile builds RustFS from source for local development and testing.

README.md

@@ -1,4 +1,4 @@
-[![RustFS](https://github.com/user-attachments/assets/1b5afcd6-a2c3-47ff-8bc3-ce882b0ddca7)](https://rustfs.com)
+[![RustFS](https://github.com/user-attachments/assets/3ba82e75-2f2d-4415-a4aa-1e4ffe9f22fd)](https://rustfs.com)
 
 <p align="center">RustFS is a high-performance, distributed object storage system built in Rust.</p>
 
@@ -42,6 +42,7 @@ Unlike other storage systems, RustFS is released under the permissible Apache 2.
 - **High Performance**: Built with Rust to ensure maximum speed and resource efficiency.
 - **Distributed Architecture**: Scalable and fault-tolerant design suitable for large-scale deployments.
 - **S3 Compatibility**: Seamless integration with existing S3-compatible applications and tools.
+- **OpenStack Keystone Integration**: Native support for OpenStack Keystone authentication with X-Auth-Token headers.
 - **Data Lake Support**: Optimized for high-throughput big data and AI workloads.
 - **Open Source**: Licensed under Apache 2.0, encouraging unrestricted community contributions and commercial usage.
 - **User-Friendly**: Designed with simplicity in mind for easy deployment and management.
@@ -54,6 +55,7 @@ Unlike other storage systems, RustFS is released under the permissible Apache 2.
 | **Logging**             | ✅ Available | **Lifecycle Management** | 🚧 Under Testing |
 | **Event Notifications** | ✅ Available | **Distributed Mode**     | 🚧 Under Testing |
 | **K8s Helm Charts**     | ✅ Available | **RustFS KMS**           | 🚧 Under Testing |
+| **Keystone Auth**       | ✅ Available | **Multi-Tenancy**        | ✅ Available     |
 
 ## RustFS vs MinIO Performance
 

crates/audit/src/factory.rs

@@ -19,8 +19,9 @@ use rumqttc::QoS;
 use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
 use rustfs_config::{
     AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT,
-    WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
+    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
+    WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
+    WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
@@ -75,6 +76,11 @@ impl TargetFactory for WebhookTargetFactory {
                 .unwrap_or(DEFAULT_LIMIT),
             client_cert: config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(),
             client_key: config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(),
+            client_ca: config.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
+            skip_tls_verify: config
+                .lookup(WEBHOOK_SKIP_TLS_VERIFY)
+                .and_then(|v| v.parse::<bool>().ok())
+                .unwrap_or(RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT),
             target_type: rustfs_targets::target::TargetType::AuditLog,
         };
 

crates/common/Cargo.toml

@@ -32,6 +32,7 @@ tokio = { workspace = true }
 tonic = { workspace = true }
 uuid = { workspace = true }
 chrono = { workspace = true }
+metrics = { workspace = true }
 rustfs-madmin = { workspace = true }
 rustfs-filemeta = { workspace = true }
 serde = { workspace = true }
@@ -39,4 +40,4 @@ path-clean = { workspace = true }
 rmp-serde = { workspace = true }
 async-trait = { workspace = true }
 s3s = { workspace = true }
-tracing = { workspace = true }
+tracing = { workspace = true }

crates/common/src/globals.rs

@@ -38,6 +38,14 @@ pub async fn set_global_local_node_name(name: &str) {
     *GLOBAL_LOCAL_NODE_NAME.write().await = name.to_string();
 }
 
+/// Get the global local node name.
+///
+/// # Returns
+/// * `String` - The local node name.
+pub async fn get_global_local_node_name() -> String {
+    GLOBAL_LOCAL_NODE_NAME.read().await.clone()
+}
+
 /// Set the global RustFS initialization time to the current UTC time.
 pub async fn set_global_init_time_now() {
     let now = Utc::now();

crates/common/src/metrics.rs

@@ -14,7 +14,8 @@
 
 use crate::last_minute::{AccElem, LastMinuteLatency};
 use chrono::{DateTime, Utc};
-use rustfs_madmin::metrics::ScannerMetrics as M_ScannerMetrics;
+use rustfs_madmin::metrics::{ScannerMetrics as M_ScannerMetrics, TimedAction};
+use serde::{Deserialize, Serialize};
 use std::{
     collections::HashMap,
     fmt::Display,
@@ -64,6 +65,37 @@ impl IlmAction {
             || *self == Self::DeleteAllVersionsAction
             || *self == Self::DelMarkerDeleteAllVersionsAction
     }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::NoneAction => "none",
+            Self::DeleteAction => "delete",
+            Self::DeleteVersionAction => "delete_version",
+            Self::TransitionAction => "transition",
+            Self::TransitionVersionAction => "transition_version",
+            Self::DeleteRestoredAction => "delete_restored",
+            Self::DeleteRestoredVersionAction => "delete_restored_version",
+            Self::DeleteAllVersionsAction => "delete_all_versions",
+            Self::DelMarkerDeleteAllVersionsAction => "del_marker_delete_all_versions",
+            Self::ActionCount => "action_count",
+        }
+    }
+
+    pub fn from_index(i: usize) -> Option<Self> {
+        match i {
+            0 => Some(Self::NoneAction),
+            1 => Some(Self::DeleteAction),
+            2 => Some(Self::DeleteVersionAction),
+            3 => Some(Self::TransitionAction),
+            4 => Some(Self::TransitionVersionAction),
+            5 => Some(Self::DeleteRestoredAction),
+            6 => Some(Self::DeleteRestoredVersionAction),
+            7 => Some(Self::DeleteAllVersionsAction),
+            8 => Some(Self::DelMarkerDeleteAllVersionsAction),
+            9 => Some(Self::ActionCount),
+            _ => None,
+        }
+    }
 }
 
 impl Display for IlmAction {
@@ -272,14 +304,76 @@ pub struct Metrics {
     cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
 }
 
-// This is a placeholder. We'll need to define this struct.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct CurrentCycle {
     pub current: u64,
+    pub next: u64,
     pub cycle_completed: Vec<DateTime<Utc>>,
     pub started: DateTime<Utc>,
 }
 
+impl CurrentCycle {
+    pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        *self = rmp_serde::from_slice(buf)?;
+        Ok(())
+    }
+
+    pub fn marshal(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
+        Ok(rmp_serde::to_vec(self)?)
+    }
+}
+
+/// OTEL metric name constants for scanner metrics
+const OTEL_SCANNER_OBJECTS_SCANNED: &str = "rustfs_scanner_objects_scanned_total";
+const OTEL_SCANNER_DIRECTORIES_SCANNED: &str = "rustfs_scanner_directories_scanned_total";
+const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
+const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
+const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
+const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
+
+/// Emit an OTEL counter increment for the given scanner metric.
+/// ScanCycle and ScanBucketDrive are handled by dedicated emit functions with labels.
+fn emit_otel_counter(metric: usize, count: u64) {
+    match Metric::from_index(metric) {
+        Some(Metric::ScanObject) => {
+            metrics::counter!(OTEL_SCANNER_OBJECTS_SCANNED).increment(count);
+        }
+        Some(Metric::ScanFolder) => {
+            metrics::counter!(OTEL_SCANNER_DIRECTORIES_SCANNED).increment(count);
+        }
+        _ => {}
+    }
+}
+
+/// Emit OTel metrics for a completed scan cycle.
+/// Counter with result label + gauge for last successful cycle duration.
+pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
+    let result = if success { "success" } else { "error" };
+    metrics::counter!(OTEL_SCANNER_CYCLES, "result" => result).increment(1);
+    if success {
+        metrics::gauge!(OTEL_SCANNER_CYCLE_DURATION_SECONDS).set(duration.as_secs_f64());
+    }
+}
+
+/// Emit OTel metrics for a completed bucket-drive scan.
+/// Counter with result/bucket/disk labels + histogram for duration.
+pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
+    let result = if success { "success" } else { "error" };
+    metrics::counter!(
+        OTEL_SCANNER_BUCKETS_SCANNED,
+        "result" => result,
+        "bucket" => bucket.to_owned(),
+        "disk" => disk.to_owned()
+    )
+    .increment(1);
+    metrics::histogram!(
+        OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS,
+        "bucket" => bucket.to_owned(),
+        "disk" => disk.to_owned()
+    )
+    .record(duration.as_secs_f64());
+}
+
 impl Metrics {
     pub fn new() -> Self {
         let operations = (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect();
@@ -307,6 +401,7 @@ impl Metrics {
 
             // Update operation count
             global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric, 1);
 
             // Update latency for realtime metrics (spawn async task for this)
             if (metric) < Metric::LastRealtime as usize {
@@ -332,6 +427,7 @@ impl Metrics {
 
             // Update operation count
             global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric, 1);
 
             // Update latency for realtime metrics with size (spawn async task)
             if (metric) < Metric::LastRealtime as usize {
@@ -352,6 +448,7 @@ impl Metrics {
 
             // Update operation count
             global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric, 1);
 
             // Update latency for realtime metrics (spawn async task)
             if (metric) < Metric::LastRealtime as usize {
@@ -373,6 +470,7 @@ impl Metrics {
 
                 // Update operation count
                 global_metrics().operations[metric].fetch_add(count as u64, Ordering::Relaxed);
+                emit_otel_counter(metric, count as u64);
 
                 // Update latency for realtime metrics (spawn async task)
                 if (metric) < Metric::LastRealtime as usize {
@@ -408,6 +506,7 @@ impl Metrics {
         let metric = metric as usize;
         // Update operation count
         global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
+        emit_otel_counter(metric, 1);
 
         // Update latency for realtime metrics
         if (metric) < Metric::LastRealtime as usize {
@@ -494,10 +593,43 @@ impl Metrics {
         for i in 0..Metric::LastRealtime as usize {
             let last_min = self.latency[i].total().await;
             if last_min.n > 0
-                && let Some(_metric) = Metric::from_index(i)
+                && let Some(metric) = Metric::from_index(i)
             {
-                // Convert to madmin TimedAction format if needed
-                // This would require implementing the conversion
+                metrics.last_minute.actions.insert(
+                    metric.as_str().to_string(),
+                    TimedAction {
+                        count: last_min.n,
+                        acc_time: last_min.total,
+                        bytes: last_min.size,
+                    },
+                );
+            }
+        }
+
+        // Lifetime ILM operations
+        for i in 0..IlmAction::ActionCount as usize {
+            let count = self.actions[i].load(Ordering::Relaxed);
+            if count > 0
+                && let Some(action) = IlmAction::from_index(i)
+            {
+                metrics.life_time_ilm.insert(action.as_str().to_string(), count);
+            }
+        }
+
+        // Last minute ILM latency
+        for i in 0..IlmAction::ActionCount as usize {
+            let last_min = self.actions_latency[i].total().await;
+            if last_min.n > 0
+                && let Some(action) = IlmAction::from_index(i)
+            {
+                metrics.last_minute.ilm.insert(
+                    action.as_str().to_string(),
+                    TimedAction {
+                        count: last_min.n,
+                        acc_time: last_min.total,
+                        bytes: last_min.size,
+                    },
+                );
             }
         }
 
@@ -550,3 +682,31 @@ impl Default for Metrics {
         Self::new()
     }
 }
+
+pub struct CloseDiskGuard(CloseDiskFn);
+
+impl CloseDiskGuard {
+    pub fn new(close_disk: CloseDiskFn) -> Self {
+        Self(close_disk)
+    }
+
+    pub async fn close(&self) {
+        self.0().await;
+    }
+}
+
+impl Drop for CloseDiskGuard {
+    fn drop(&mut self) {
+        // Drop cannot be async, so we spawn the async cleanup task
+        // The task will run in the background and complete asynchronously
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            let close_fn = self.0.clone();
+            handle.spawn(async move {
+                close_fn().await;
+            });
+        } else {
+            // If we're not in a tokio runtime context, we can't spawn
+            // This is a best-effort cleanup, so we just skip it
+        }
+    }
+}

crates/config/README.md

@@ -32,6 +32,24 @@
 
 For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
+## Environment Variable Naming Conventions
+
+RustFS uses a flat naming style for top-level configuration: environment variables are `RUSTFS_*` without nested module segments.
+
+Examples:
+- `RUSTFS_REGION`
+- `RUSTFS_ADDRESS`
+- `RUSTFS_VOLUMES`
+- `RUSTFS_LICENSE`
+
+Current guidance:
+- Prefer module-specific names only when they are not top-level product configuration.
+- Renamed variables must keep backward-compatible aliases until before beta.
+- Alias usage must emit deprecation warnings and be treated as transitional only.
+- Deprecated example:
+  - `RUSTFS_ENABLE_SCANNER` -> `RUSTFS_SCANNER_ENABLED`
+  - `RUSTFS_ENABLE_HEAL` -> `RUSTFS_HEAL_ENABLED`
+
 ## 📄 License
 
 This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/config/src/audit/webhook.rs

@@ -20,9 +20,11 @@ pub const ENV_AUDIT_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_WEBHOOK_QUEUE_LIMI
 pub const ENV_AUDIT_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR";
 pub const ENV_AUDIT_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_CERT";
 pub const ENV_AUDIT_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_KEY";
+pub const ENV_AUDIT_WEBHOOK_CLIENT_CA: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_CA";
+pub const ENV_AUDIT_WEBHOOK_SKIP_TLS_VERIFY: &str = "RUSTFS_AUDIT_WEBHOOK_SKIP_TLS_VERIFY";
 
 /// List of all environment variable keys for a webhook target.
-pub const ENV_AUDIT_WEBHOOK_KEYS: &[&str; 7] = &[
+pub const ENV_AUDIT_WEBHOOK_KEYS: &[&str; 9] = &[
     ENV_AUDIT_WEBHOOK_ENABLE,
     ENV_AUDIT_WEBHOOK_ENDPOINT,
     ENV_AUDIT_WEBHOOK_AUTH_TOKEN,
@@ -30,6 +32,8 @@ pub const ENV_AUDIT_WEBHOOK_KEYS: &[&str; 7] = &[
     ENV_AUDIT_WEBHOOK_QUEUE_DIR,
     ENV_AUDIT_WEBHOOK_CLIENT_CERT,
     ENV_AUDIT_WEBHOOK_CLIENT_KEY,
+    ENV_AUDIT_WEBHOOK_CLIENT_CA,
+    ENV_AUDIT_WEBHOOK_SKIP_TLS_VERIFY,
 ];
 
 /// A list of all valid configuration keys for a webhook target.
@@ -42,4 +46,6 @@ pub const AUDIT_WEBHOOK_KEYS: &[&str] = &[
     crate::WEBHOOK_CLIENT_CERT,
     crate::WEBHOOK_CLIENT_KEY,
     crate::COMMENT_KEY,
+    crate::WEBHOOK_CLIENT_CA,
+    crate::WEBHOOK_SKIP_TLS_VERIFY,
 ];

crates/config/src/constants/app.rs

@@ -98,12 +98,39 @@ pub const RUSTFS_HTTP_PREFIX: &str = "http://";
 /// Default value: https://
 pub const RUSTFS_HTTPS_PREFIX: &str = "https://";
 
+/// Default documentation URL for rustfs
+/// This is the default documentation URL for rustfs.
+/// It is used to provide the documentation of the application.
+/// Default value: https://docs.rustfs.com
+pub const RUSTFS_DOCS_URL: &str = "https://docs.rustfs.com";
+
+/// Default GitHub URL for rustfs
+/// This is the default GitHub URL for rustfs.
+/// It is used to provide the source code of the application.
+/// Default value: https://github.com/rustfs/rustfs
+pub const RUSTFS_GITHUB_URL: &str = "https://github.com/rustfs/rustfs";
+
+/// Default license for rustfs
+/// This is the default license for rustfs.
+/// It is used to provide the license of the application.
+/// Default value: Apache-2.0
+pub const RUSTFS_LICENSE: &str = "Apache-2.0";
+
+/// Default license URL for rustfs
+/// This is the default license URL for rustfs.
+/// It is used to provide the license URL of the application.
+/// Default value: https://www.apache.org/licenses/LICENSE-2.0
+pub const RUSTFS_LICENSE_URL: &str = "https://www.apache.org/licenses/LICENSE-2.0";
+
 /// Environment variable for rustfs address
 /// This is the environment variable for rustfs address.
 /// It is used to bind the server to a specific address.
 /// Example: RUSTFS_ADDRESS=":9000"
 pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 
+/// Environment variable for server volumes.
+pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
+
 /// Default port for rustfs
 /// This is the default port for rustfs.
 /// This is used to bind the server to a specific port.
@@ -119,20 +146,34 @@ pub const DEFAULT_CONSOLE_PORT: u16 = 9001;
 
 /// Default address for rustfs console
 /// This is the default address for rustfs console.
+/// This is used to bind the console server to a specific address.
+/// Default value: :9001
 pub const DEFAULT_CONSOLE_ADDRESS: &str = concat!(":", DEFAULT_CONSOLE_PORT);
 
+/// Default region for rustfs
+/// This is the default region for rustfs.
+/// It is used to identify the region of the application.
+/// Default value: us-east-1
+pub const RUSTFS_REGION: &str = "us-east-1";
+
+/// Environment variable for server region.
+pub const ENV_RUSTFS_REGION: &str = "RUSTFS_REGION";
+
+/// Environment variable for server license.
+pub const ENV_RUSTFS_LICENSE: &str = "RUSTFS_LICENSE";
+
 /// Default log filename for rustfs
 /// This is the default log filename for rustfs.
 /// It is used to store the logs of the application.
 /// Default value: rustfs.log
-/// Environment variable: RUSTFS_OBSERVABILITY_LOG_FILENAME
+/// Environment variable: RUSTFS_OBS_LOG_FILENAME
 pub const DEFAULT_LOG_FILENAME: &str = "rustfs";
 
 /// Default OBS log filename for rustfs
 /// This is the default log filename for OBS.
 /// It is used to store the logs of the application.
 /// Default value: rustfs.log
-pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, "");
+pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, ".log");
 
 /// Default log directory for rustfs
 /// This is the default log directory for rustfs.
@@ -151,9 +192,9 @@ pub const DEFAULT_LOG_ROTATION_SIZE_MB: u64 = 100;
 /// Default log rotation time for rustfs
 /// This is the default log rotation time for rustfs.
 /// It is used to rotate the logs of the application.
-/// Default value: hour, eg: day,hour,minute,second
+/// Default value: hour, eg: daily,hourly,minutely
 /// Environment variable: RUSTFS_OBS_LOG_ROTATION_TIME
-pub const DEFAULT_LOG_ROTATION_TIME: &str = "hour";
+pub const DEFAULT_LOG_ROTATION_TIME: &str = "hourly";
 
 /// Default log keep files for rustfs
 /// This is the default log keep files for rustfs.
@@ -162,11 +203,29 @@ pub const DEFAULT_LOG_ROTATION_TIME: &str = "hour";
 /// Environment variable: RUSTFS_OBS_LOG_KEEP_FILES
 pub const DEFAULT_LOG_KEEP_FILES: usize = 30;
 
+/// Default trace export enabled
+/// It is used to enable or disable exporting traces
+/// Default value: true
+/// Environment variable: RUSTFS_OBS_TRACES_EXPORT_ENABLED
+pub const DEFAULT_OBS_TRACES_EXPORT_ENABLED: bool = true;
+
+/// Default metrics export enabled
+/// It is used to enable or disable exporting metrics
+/// Default value: true
+/// Environment variable: RUSTFS_OBS_METRICS_EXPORT_ENABLED
+pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true;
+
+/// Default logs export enabled
+/// It is used to enable or disable exporting logs
+/// Default value: true
+/// Environment variable: RUSTFS_OBS_LOGS_EXPORT_ENABLED
+pub const DEFAULT_OBS_LOGS_EXPORT_ENABLED: bool = true;
+
 /// Default log local logging enabled for rustfs
 /// This is the default log local logging enabled for rustfs.
 /// It is used to enable or disable local logging of the application.
 /// Default value: false
-/// Environment variable: RUSTFS_OBS_LOGL_STDOUT_ENABLED
+/// Environment variable: RUSTFS_OBS_LOG_STDOUT_ENABLED
 pub const DEFAULT_OBS_LOG_STDOUT_ENABLED: bool = false;
 
 /// Constant representing 1 Kibibyte (1024 bytes)

crates/config/src/constants/env.rs

@@ -20,6 +20,8 @@ pub const EVENT_DEFAULT_DIR: &str = "/opt/rustfs/events"; // Default directory f
 pub const AUDIT_DEFAULT_DIR: &str = "/opt/rustfs/audit"; // Default directory for audit store
 pub const DEFAULT_LIMIT: u64 = 100000; // Default store limit
 
+pub const RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT: bool = false;
+
 /// Standard config keys and values.
 pub const ENABLE_KEY: &str = "enable";
 pub const COMMENT_KEY: &str = "comment";

crates/config/src/constants/mod.rs

@@ -19,6 +19,7 @@ pub(crate) mod console;
 pub(crate) mod env;
 pub(crate) mod heal;
 pub(crate) mod object;
+pub(crate) mod oidc;
 pub(crate) mod profiler;
 pub(crate) mod protocols;
 pub(crate) mod proxy;

crates/config/src/constants/oidc.rs

@@ -0,0 +1,91 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// OIDC configuration field keys (used in KVS)
+pub const OIDC_CONFIG_URL: &str = "config_url";
+pub const OIDC_CLIENT_ID: &str = "client_id";
+pub const OIDC_CLIENT_SECRET: &str = "client_secret";
+pub const OIDC_SCOPES: &str = "scopes";
+pub const OIDC_REDIRECT_URI: &str = "redirect_uri";
+pub const OIDC_REDIRECT_URI_DYNAMIC: &str = "redirect_uri_dynamic";
+pub const OIDC_CLAIM_NAME: &str = "claim_name";
+pub const OIDC_CLAIM_PREFIX: &str = "claim_prefix";
+pub const OIDC_ROLE_POLICY: &str = "role_policy";
+pub const OIDC_DISPLAY_NAME: &str = "display_name";
+pub const OIDC_GROUPS_CLAIM: &str = "groups_claim";
+pub const OIDC_EMAIL_CLAIM: &str = "email_claim";
+pub const OIDC_USERNAME_CLAIM: &str = "username_claim";
+
+// Environment variable names for OIDC
+pub const ENV_IDENTITY_OPENID_ENABLE: &str = "RUSTFS_IDENTITY_OPENID_ENABLE";
+pub const ENV_IDENTITY_OPENID_CONFIG_URL: &str = "RUSTFS_IDENTITY_OPENID_CONFIG_URL";
+pub const ENV_IDENTITY_OPENID_CLIENT_ID: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_ID";
+pub const ENV_IDENTITY_OPENID_CLIENT_SECRET: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_SECRET";
+pub const ENV_IDENTITY_OPENID_SCOPES: &str = "RUSTFS_IDENTITY_OPENID_SCOPES";
+pub const ENV_IDENTITY_OPENID_REDIRECT_URI: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI";
+pub const ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC";
+pub const ENV_IDENTITY_OPENID_CLAIM_NAME: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_NAME";
+pub const ENV_IDENTITY_OPENID_CLAIM_PREFIX: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_PREFIX";
+pub const ENV_IDENTITY_OPENID_ROLE_POLICY: &str = "RUSTFS_IDENTITY_OPENID_ROLE_POLICY";
+pub const ENV_IDENTITY_OPENID_DISPLAY_NAME: &str = "RUSTFS_IDENTITY_OPENID_DISPLAY_NAME";
+pub const ENV_IDENTITY_OPENID_GROUPS_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM";
+pub const ENV_IDENTITY_OPENID_EMAIL_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM";
+pub const ENV_IDENTITY_OPENID_USERNAME_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM";
+
+/// List of all environment variable keys for an OIDC provider.
+pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
+    ENV_IDENTITY_OPENID_ENABLE,
+    ENV_IDENTITY_OPENID_CONFIG_URL,
+    ENV_IDENTITY_OPENID_CLIENT_ID,
+    ENV_IDENTITY_OPENID_CLIENT_SECRET,
+    ENV_IDENTITY_OPENID_SCOPES,
+    ENV_IDENTITY_OPENID_REDIRECT_URI,
+    ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC,
+    ENV_IDENTITY_OPENID_CLAIM_NAME,
+    ENV_IDENTITY_OPENID_CLAIM_PREFIX,
+    ENV_IDENTITY_OPENID_ROLE_POLICY,
+    ENV_IDENTITY_OPENID_DISPLAY_NAME,
+    ENV_IDENTITY_OPENID_GROUPS_CLAIM,
+    ENV_IDENTITY_OPENID_EMAIL_CLAIM,
+    ENV_IDENTITY_OPENID_USERNAME_CLAIM,
+];
+
+/// A list of all valid configuration keys for an OIDC provider.
+pub const IDENTITY_OPENID_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    OIDC_CONFIG_URL,
+    OIDC_CLIENT_ID,
+    OIDC_CLIENT_SECRET,
+    OIDC_SCOPES,
+    OIDC_REDIRECT_URI,
+    OIDC_REDIRECT_URI_DYNAMIC,
+    OIDC_CLAIM_NAME,
+    OIDC_CLAIM_PREFIX,
+    OIDC_ROLE_POLICY,
+    OIDC_DISPLAY_NAME,
+    OIDC_GROUPS_CLAIM,
+    OIDC_EMAIL_CLAIM,
+    OIDC_USERNAME_CLAIM,
+    crate::COMMENT_KEY,
+];
+
+// Default values
+pub const OIDC_DEFAULT_SCOPES: &str = "openid,profile,email";
+pub const OIDC_DEFAULT_CLAIM_NAME: &str = "groups";
+pub const OIDC_DEFAULT_GROUPS_CLAIM: &str = "groups";
+pub const OIDC_DEFAULT_EMAIL_CLAIM: &str = "email";
+pub const OIDC_DEFAULT_USERNAME_CLAIM: &str = "preferred_username";
+
+// Subsystem identifier
+pub const IDENTITY_OPENID_SUB_SYS: &str = "identity_openid";

crates/config/src/constants/targets.rs

@@ -16,6 +16,8 @@ pub const WEBHOOK_ENDPOINT: &str = "endpoint";
 pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token";
 pub const WEBHOOK_CLIENT_CERT: &str = "client_cert";
 pub const WEBHOOK_CLIENT_KEY: &str = "client_key";
+pub const WEBHOOK_CLIENT_CA: &str = "client_ca";
+pub const WEBHOOK_SKIP_TLS_VERIFY: &str = "skip_tls_verify";
 pub const WEBHOOK_BATCH_SIZE: &str = "batch_size";
 pub const WEBHOOK_QUEUE_LIMIT: &str = "queue_limit";
 pub const WEBHOOK_QUEUE_DIR: &str = "queue_dir";

crates/config/src/lib.rs

@@ -44,6 +44,10 @@ pub use constants::scanner::*;
 pub use constants::targets::*;
 #[cfg(feature = "constants")]
 pub use constants::tls::*;
+#[cfg(feature = "constants")]
+pub mod oidc {
+    pub use super::constants::oidc::*;
+}
 #[cfg(feature = "audit")]
 pub mod audit;
 #[cfg(feature = "notify")]

crates/config/src/notify/webhook.rs

@@ -22,6 +22,8 @@ pub const NOTIFY_WEBHOOK_KEYS: &[&str] = &[
     crate::WEBHOOK_CLIENT_CERT,
     crate::WEBHOOK_CLIENT_KEY,
     crate::COMMENT_KEY,
+    crate::WEBHOOK_CLIENT_CA,
+    crate::WEBHOOK_SKIP_TLS_VERIFY,
 ];
 
 // Webhook Environment Variables
@@ -32,8 +34,10 @@ pub const ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_LI
 pub const ENV_NOTIFY_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR";
 pub const ENV_NOTIFY_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CERT";
 pub const ENV_NOTIFY_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_KEY";
+pub const ENV_NOTIFY_WEBHOOK_CLIENT_CA: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CA";
+pub const ENV_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY: &str = "RUSTFS_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY";
 
-pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 7] = &[
+pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 9] = &[
     ENV_NOTIFY_WEBHOOK_ENABLE,
     ENV_NOTIFY_WEBHOOK_ENDPOINT,
     ENV_NOTIFY_WEBHOOK_AUTH_TOKEN,
@@ -41,4 +45,6 @@ pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 7] = &[
     ENV_NOTIFY_WEBHOOK_QUEUE_DIR,
     ENV_NOTIFY_WEBHOOK_CLIENT_CERT,
     ENV_NOTIFY_WEBHOOK_CLIENT_KEY,
+    ENV_NOTIFY_WEBHOOK_CLIENT_CA,
+    ENV_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY,
 ];

crates/config/src/observability/mod.rs

@@ -15,6 +15,7 @@
 // Observability Keys
 
 mod metrics;
+use const_str::concat;
 pub use metrics::*;
 
 pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
@@ -27,6 +28,12 @@ pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
 pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
 pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
 pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
+
+/// Per-signal enable/disable flags (default: true)
+pub const ENV_OBS_TRACES_EXPORT_ENABLED: &str = "RUSTFS_OBS_TRACES_EXPORT_ENABLED";
+pub const ENV_OBS_METRICS_EXPORT_ENABLED: &str = "RUSTFS_OBS_METRICS_EXPORT_ENABLED";
+pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED";
+
 pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
 pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
 pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
@@ -35,23 +42,32 @@ pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB"
 pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
 pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
 
-/// Log pool capacity for async logging
-pub const ENV_OBS_LOG_POOL_CAPA: &str = "RUSTFS_OBS_LOG_POOL_CAPA";
+/// Log cleanup related configurations
+pub const ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES";
+pub const ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES";
+pub const ENV_OBS_LOG_COMPRESS_OLD_FILES: &str = "RUSTFS_OBS_LOG_COMPRESS_OLD_FILES";
+pub const ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL: &str = "RUSTFS_OBS_LOG_GZIP_COMPRESSION_LEVEL";
+pub const ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS: &str = "RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS";
+pub const ENV_OBS_LOG_EXCLUDE_PATTERNS: &str = "RUSTFS_OBS_LOG_EXCLUDE_PATTERNS";
+pub const ENV_OBS_LOG_DELETE_EMPTY_FILES: &str = "RUSTFS_OBS_LOG_DELETE_EMPTY_FILES";
+pub const ENV_OBS_LOG_MIN_FILE_AGE_SECONDS: &str = "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS";
+pub const ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS: &str = "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS";
+pub const ENV_OBS_LOG_DRY_RUN: &str = "RUSTFS_OBS_LOG_DRY_RUN";
+pub const ENV_OBS_LOG_MATCH_MODE: &str = "RUSTFS_OBS_LOG_MATCH_MODE";
 
-/// Log message capacity for async logging
-pub const ENV_OBS_LOG_MESSAGE_CAPA: &str = "RUSTFS_OBS_LOG_MESSAGE_CAPA";
-
-/// Log flush interval in milliseconds for async logging
-pub const ENV_OBS_LOG_FLUSH_MS: &str = "RUSTFS_OBS_LOG_FLUSH_MS";
-
-/// Default values for log pool
-pub const DEFAULT_OBS_LOG_POOL_CAPA: usize = 10240;
-
-/// Default values for message capacity
-pub const DEFAULT_OBS_LOG_MESSAGE_CAPA: usize = 32768;
-
-/// Default values for flush interval in milliseconds
-pub const DEFAULT_OBS_LOG_FLUSH_MS: u64 = 200;
+/// Default values for log cleanup
+pub const DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES: u64 = 2 * 1024 * 1024 * 1024; // 2 GiB
+pub const DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: u64 = 0; // No single file limit
+pub const DEFAULT_OBS_LOG_COMPRESS_OLD_FILES: bool = true;
+pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL: u32 = 6;
+pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_EXTENSION: &str = "gz";
+pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION: &str = concat!(".", DEFAULT_OBS_LOG_GZIP_COMPRESSION_EXTENSION);
+pub const DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS: u64 = 30;
+pub const DEFAULT_OBS_LOG_DELETE_EMPTY_FILES: bool = true;
+pub const DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS: u64 = 3600; // 1 hour
+pub const DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS: u64 = 1800; // 0.5 hours
+pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false;
+pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix";
 
 /// Default values for observability configuration
 // ### Supported Environment Values
@@ -87,6 +103,24 @@ mod tests {
         assert_eq!(ENV_OBS_LOG_ROTATION_SIZE_MB, "RUSTFS_OBS_LOG_ROTATION_SIZE_MB");
         assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
         assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
+        assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED");
+        assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
+        assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
+        // Test log cleanup related env keys
+        assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
+        assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
+        assert_eq!(ENV_OBS_LOG_COMPRESS_OLD_FILES, "RUSTFS_OBS_LOG_COMPRESS_OLD_FILES");
+        assert_eq!(ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, "RUSTFS_OBS_LOG_GZIP_COMPRESSION_LEVEL");
+        assert_eq!(
+            ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
+            "RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS"
+        );
+        assert_eq!(ENV_OBS_LOG_EXCLUDE_PATTERNS, "RUSTFS_OBS_LOG_EXCLUDE_PATTERNS");
+        assert_eq!(ENV_OBS_LOG_DELETE_EMPTY_FILES, "RUSTFS_OBS_LOG_DELETE_EMPTY_FILES");
+        assert_eq!(ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS");
+        assert_eq!(ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS");
+        assert_eq!(ENV_OBS_LOG_DRY_RUN, "RUSTFS_OBS_LOG_DRY_RUN");
+        assert_eq!(ENV_OBS_LOG_MATCH_MODE, "RUSTFS_OBS_LOG_MATCH_MODE");
     }
 
     #[test]
@@ -95,5 +129,6 @@ mod tests {
         assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
         assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
         assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
+        assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix");
     }
 }

crates/e2e_test/Cargo.toml

@@ -59,5 +59,4 @@ sha2 = { workspace = true }
 suppaftp = { workspace = true, features = ["tokio", "rustls-aws-lc-rs"] }
 rcgen.workspace = true
 anyhow.workspace = true
-rustls.workspace = true
-rustls-pemfile.workspace = true
+rustls.workspace = true

crates/e2e_test/src/anonymous_access_test.rs

@@ -0,0 +1,172 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression tests for Issue #2036
+//! Verifies that anonymous access works correctly with bucket policies
+//! when PublicAccessBlock configuration is missing or explicitly set.
+
+use crate::common::{RustFSTestEnvironment, init_logging};
+use aws_sdk_s3::types::PublicAccessBlockConfiguration;
+use serial_test::serial;
+use tracing::info;
+
+async fn setup_public_bucket(
+    env: &RustFSTestEnvironment,
+    bucket_name: &str,
+) -> Result<aws_sdk_s3::Client, Box<dyn std::error::Error + Send + Sync>> {
+    let admin_client = env.create_s3_client();
+
+    admin_client.create_bucket().bucket(bucket_name).send().await?;
+
+    let policy_json = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Sid": "AllowAnonymousGetObject",
+                "Effect": "Allow",
+                "Principal": "*",
+                "Action": ["s3:GetObject"],
+                "Resource": [format!("arn:aws:s3:::{}/*", bucket_name)]
+            }
+        ]
+    })
+    .to_string();
+
+    admin_client
+        .put_bucket_policy()
+        .bucket(bucket_name)
+        .policy(&policy_json)
+        .send()
+        .await?;
+
+    admin_client
+        .put_object()
+        .bucket(bucket_name)
+        .key("test.txt")
+        .body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello anonymous"))
+        .send()
+        .await?;
+
+    Ok(admin_client)
+}
+
+async fn anonymous_get_object(
+    env: &RustFSTestEnvironment,
+    bucket_name: &str,
+    key: &str,
+) -> Result<reqwest::Response, reqwest::Error> {
+    let url = format!("{}/{}/{}", env.url, bucket_name, key);
+    reqwest::Client::new().get(&url).send().await
+}
+
+/// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it
+/// and no PublicAccessBlock configuration exists (ConfigNotFound).
+#[tokio::test]
+#[serial]
+async fn test_anonymous_access_allowed_when_public_access_block_missing() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
+{
+    init_logging();
+    info!("Starting test: anonymous access with missing PublicAccessBlock config...");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket_name = "anon-test-no-pab";
+    let admin_client = setup_public_bucket(&env, bucket_name).await?;
+
+    let _ = admin_client.delete_public_access_block().bucket(bucket_name).send().await;
+
+    let resp = anonymous_get_object(&env, bucket_name, "test.txt").await?;
+    assert_eq!(
+        resp.status().as_u16(),
+        200,
+        "Anonymous GetObject should succeed when no PublicAccessBlock config exists"
+    );
+
+    info!("Test passed: anonymous access allowed with missing PublicAccessBlock config");
+    Ok(())
+}
+
+/// Anonymous GetObject should be denied when RestrictPublicBuckets is true.
+#[tokio::test]
+#[serial]
+async fn test_anonymous_access_denied_when_restrict_public_buckets_enabled()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    info!("Starting test: anonymous access denied with RestrictPublicBuckets=true...");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket_name = "anon-test-restrict";
+    let admin_client = setup_public_bucket(&env, bucket_name).await?;
+
+    admin_client
+        .put_public_access_block()
+        .bucket(bucket_name)
+        .public_access_block_configuration(
+            PublicAccessBlockConfiguration::builder()
+                .restrict_public_buckets(true)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let resp = anonymous_get_object(&env, bucket_name, "test.txt").await?;
+    assert_eq!(
+        resp.status().as_u16(),
+        403,
+        "Anonymous GetObject should be denied when RestrictPublicBuckets is true"
+    );
+
+    info!("Test passed: anonymous access denied with RestrictPublicBuckets=true");
+    Ok(())
+}
+
+/// Anonymous GetObject should succeed when PublicAccessBlock exists but
+/// RestrictPublicBuckets is explicitly false.
+#[tokio::test]
+#[serial]
+async fn test_anonymous_access_allowed_when_restrict_public_buckets_disabled()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    info!("Starting test: anonymous access allowed with RestrictPublicBuckets=false...");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket_name = "anon-test-allow";
+    let admin_client = setup_public_bucket(&env, bucket_name).await?;
+
+    admin_client
+        .put_public_access_block()
+        .bucket(bucket_name)
+        .public_access_block_configuration(
+            PublicAccessBlockConfiguration::builder()
+                .restrict_public_buckets(false)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let resp = anonymous_get_object(&env, bucket_name, "test.txt").await?;
+    assert_eq!(
+        resp.status().as_u16(),
+        200,
+        "Anonymous GetObject should succeed when RestrictPublicBuckets is false"
+    );
+
+    info!("Test passed: anonymous access allowed with RestrictPublicBuckets=false");
+    Ok(())
+}

crates/e2e_test/src/content_encoding_test.rs

@@ -82,4 +82,60 @@ mod tests {
 
         env.stop_server();
     }
+
+    /// Issue #1857: Content-Encoding "aws-chunked" is used by SigV4 streaming clients and must
+    /// not be stored or returned. Upload with aws-chunked and verify GET/HEAD do not return it.
+    #[tokio::test]
+    #[serial]
+    async fn test_content_encoding_aws_chunked_not_returned_issue_1857() {
+        init_logging();
+        info!("Issue #1857: aws-chunked must not be persisted or returned");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "content-encoding-aws-chunked-test";
+        let key = "streamed/object.bin";
+        let content = b"streaming upload body";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .content_encoding("aws-chunked")
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("PUT failed");
+
+        let get_resp = client.get_object().bucket(bucket).key(key).send().await.expect("GET failed");
+        assert_eq!(
+            get_resp.content_encoding(),
+            None,
+            "GET must not return Content-Encoding when only aws-chunked was sent (Issue #1857)"
+        );
+
+        let head_resp = client
+            .head_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("HEAD failed");
+        assert_eq!(
+            head_resp.content_encoding(),
+            None,
+            "HEAD must not return Content-Encoding when only aws-chunked was sent (Issue #1857)"
+        );
+
+        env.stop_server();
+    }
 }

crates/e2e_test/src/delete_objects_versioning_test.rs

@@ -0,0 +1,288 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for Issue #1878:
+//! "In a versioned Bucket, DeleteMarkers are not appearing straight after
+//! a delete_objects is called."
+//!
+//! Root cause: `delete_versions_internal` wrote new xl.meta to disk via
+//! `write_all_private` without invalidating the `GlobalFileCache`. Subsequent
+//! calls to `read_metadata` returned the stale cached xl.meta (without the
+//! delete marker), making `list_object_versions` show the old version as
+//! `IsLatest=true` rather than the new delete marker.
+//!
+//! Fix: `write_all_private` now calls `get_global_file_cache().invalidate()`
+//! after every successful write, and `rename_data` also invalidates the cache
+//! for the destination path after the atomic rename.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
+    use serial_test::serial;
+    use tracing::info;
+
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    /// Regression test for Issue #1878.
+    ///
+    /// Verifies that after calling `delete_objects` (the batch plural API) on
+    /// a versioned bucket, calling `list_object_versions` **immediately** (with
+    /// no sleep) returns the newly-created DeleteMarker with `is_latest = true`.
+    #[tokio::test]
+    #[serial]
+    async fn test_delete_objects_delete_marker_immediately_visible() {
+        init_logging();
+        info!("🧪 TEST: DeleteMarker from delete_objects is immediately visible via list_object_versions");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-delete-marker-visibility";
+        let key = "test-prefix/test-object.txt";
+
+        // 1. Create bucket
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+        info!("✅ Bucket created: {}", bucket);
+
+        // 2. Enable versioning
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to enable versioning");
+        info!("✅ Versioning enabled");
+
+        // 3. Put an object
+        let put_resp = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello versioned world\n"))
+            .send()
+            .await
+            .expect("Failed to put object");
+        let original_version_id = put_resp.version_id().map(str::to_string);
+        info!("✅ Object put, version_id: {:?}", original_version_id);
+        assert!(
+            original_version_id.is_some(),
+            "PutObject should return a version_id in a versioned bucket"
+        );
+
+        // 4. List versions – should show exactly 1 Version with is_latest = true
+        let list_before = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions before delete");
+
+        let versions_before = list_before.versions().to_vec();
+        let markers_before = list_before.delete_markers().to_vec();
+        assert_eq!(versions_before.len(), 1, "Should have 1 version before delete");
+        assert!(markers_before.is_empty(), "Should have no delete markers before delete");
+        assert_eq!(
+            versions_before[0].is_latest(),
+            Some(true),
+            "Original version should be latest before delete"
+        );
+        info!("✅ Verified: 1 version, 0 delete markers before delete_objects");
+
+        // 5. Call delete_objects (plural, no version_id → creates a delete marker)
+        let del_resp = client
+            .delete_objects()
+            .bucket(bucket)
+            .delete(
+                Delete::builder()
+                    .objects(
+                        ObjectIdentifier::builder()
+                            .key(key)
+                            .build()
+                            .expect("Failed to build object identifier"),
+                    )
+                    .build()
+                    .expect("Failed to build delete request"),
+            )
+            .send()
+            .await
+            .expect("Failed to call delete_objects");
+
+        let deleted = del_resp.deleted();
+        assert_eq!(deleted.len(), 1, "delete_objects should report 1 deleted object");
+        let delete_marker_version_id = deleted[0].delete_marker_version_id().map(str::to_string);
+        assert!(
+            deleted[0].delete_marker() == Some(true),
+            "delete_objects should report delete_marker=true for versioned delete"
+        );
+        assert!(
+            delete_marker_version_id.is_some(),
+            "delete_objects should return a delete_marker_version_id"
+        );
+        info!("✅ delete_objects succeeded, delete_marker_version_id: {:?}", delete_marker_version_id);
+
+        // 6. List versions IMMEDIATELY (no sleep!) – this is the key regression assertion.
+        //    Before the fix, the stale file cache caused the listing to return the original
+        //    version as is_latest=true without any delete marker.
+        let list_after = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions immediately after delete_objects");
+
+        let versions_after = list_after.versions().to_vec();
+        let markers_after = list_after.delete_markers().to_vec();
+
+        info!("Versions after delete_objects: {:?}", versions_after.len());
+        info!("Delete markers after delete_objects: {:?}", markers_after.len());
+
+        // The delete marker must be visible immediately – no delay required.
+        assert_eq!(
+            markers_after.len(),
+            1,
+            "list_object_versions should return exactly 1 delete marker IMMEDIATELY after delete_objects (regression #1878)"
+        );
+        assert_eq!(markers_after[0].is_latest(), Some(true), "The delete marker should be is_latest=true");
+        assert_eq!(
+            markers_after[0].version_id().map(str::to_string),
+            delete_marker_version_id,
+            "The delete marker version_id should match the one returned by delete_objects"
+        );
+
+        // The original version should still be present, but is_latest = false
+        assert_eq!(versions_after.len(), 1, "The original version should still be present");
+        assert_eq!(
+            versions_after[0].is_latest(),
+            Some(false),
+            "Original version should be is_latest=false after delete marker is created"
+        );
+
+        info!("✅ TEST PASSED: Delete marker is immediately visible after delete_objects (no sleep needed)");
+    }
+
+    /// Additional regression check: verify that multiple objects deleted via
+    /// a single `delete_objects` call all have their delete markers visible
+    /// immediately afterwards.
+    #[tokio::test]
+    #[serial]
+    async fn test_delete_objects_multiple_keys_delete_markers_immediately_visible() {
+        init_logging();
+        info!("🧪 TEST: Multiple delete markers from delete_objects are immediately visible");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-multi-delete-marker-visibility";
+        let keys = ["obj1.txt", "obj2.txt", "obj3.txt"];
+
+        // Create bucket and enable versioning
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to enable versioning");
+
+        // Put all objects
+        for key in &keys {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(*key)
+                .body(aws_sdk_s3::primitives::ByteStream::from_static(b"data"))
+                .send()
+                .await
+                .unwrap_or_else(|_| panic!("Failed to put object {}", key));
+        }
+        info!("✅ Put {} objects", keys.len());
+
+        // Delete all objects with a single delete_objects call
+        let identifiers: Vec<ObjectIdentifier> = keys
+            .iter()
+            .map(|k| {
+                ObjectIdentifier::builder()
+                    .key(*k)
+                    .build()
+                    .expect("Failed to build ObjectIdentifier")
+            })
+            .collect();
+
+        let del_resp = client
+            .delete_objects()
+            .bucket(bucket)
+            .delete(
+                Delete::builder()
+                    .set_objects(Some(identifiers))
+                    .build()
+                    .expect("Failed to build delete"),
+            )
+            .send()
+            .await
+            .expect("Failed to call delete_objects");
+
+        assert_eq!(del_resp.deleted().len(), keys.len(), "All keys should be reported deleted");
+        info!("✅ delete_objects deleted {} objects", keys.len());
+
+        // Immediately list all versions and check each key has a delete marker
+        let list_resp = client
+            .list_object_versions()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to list object versions");
+
+        let markers = list_resp.delete_markers().to_vec();
+        assert_eq!(
+            markers.len(),
+            keys.len(),
+            "Each deleted key should have an immediately-visible delete marker (regression #1878)"
+        );
+
+        for marker in &markers {
+            assert_eq!(marker.is_latest(), Some(true), "Each delete marker should be is_latest=true");
+        }
+
+        info!("✅ TEST PASSED: All {} delete markers are immediately visible", keys.len());
+    }
+}

crates/e2e_test/src/group_delete_test.rs

@@ -0,0 +1,208 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! E2E tests for group management (fixes #2028).
+
+use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_put, init_logging};
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::{Client, Config};
+use serial_test::serial;
+use tracing::info;
+
+fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-group-test");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+/// Test that deleting a group with members fails, and deleting an empty group succeeds.
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+#[ignore = "requires awscurl and spawns a real RustFS server"]
+async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    // 1. Create a user
+    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
+    let user_body = serde_json::json!({
+        "secretKey": "testuser1secret",
+        "status": "enabled"
+    });
+    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created testuser1");
+
+    // 2. Create a group with testuser1 as a member
+    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let add_member_body = serde_json::json!({
+        "group": "testgroup",
+        "members": ["testuser1"],
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Added testuser1 to testgroup");
+
+    // 3. Attempt to delete the group while it still has members — should fail
+    let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
+    let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
+    assert!(delete_result.is_err(), "deleting a non-empty group should fail");
+    info!("Delete of non-empty group correctly rejected");
+
+    // 4. Remove the member from the group
+    let remove_member_body = serde_json::json!({
+        "group": "testgroup",
+        "members": ["testuser1"],
+        "isRemove": true,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Removed testuser1 from testgroup");
+
+    // 5. Delete the now-empty group — should succeed
+    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    info!("Deleted empty testgroup successfully");
+
+    // 6. Verify the group no longer exists
+    let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
+    let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
+    assert!(get_result.is_err(), "group should no longer exist after deletion");
+    info!("Confirmed testgroup no longer exists");
+
+    Ok(())
+}
+
+/// Test that a user with only group membership (no explicit user policy) gets group policies
+/// and can perform actions allowed by the group (regression test for #2028.1).
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+#[ignore = "requires awscurl and spawns a real RustFS server"]
+async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let user_name = "grouponlyuser";
+    let user_secret = "grouponlysecret";
+    let group_name = "policygroup";
+    let policy_name = "ListBucketsOnlyPolicy";
+
+    // 1. Create canned policy that allows ListAllMyBuckets only
+    let policy_doc = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:ListAllMyBuckets"],
+            "Resource": ["*"]
+        }]
+    });
+    let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created canned policy {}", policy_name);
+
+    // 2. Create user with no explicit policy
+    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
+    let user_body = serde_json::json!({
+        "secretKey": user_secret,
+        "status": "enabled"
+    });
+    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created user {} with no explicit policy", user_name);
+
+    // 3. Add user to group (creates group with this member; user_group_memberships must be updated)
+    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let add_member_body = serde_json::json!({
+        "group": group_name,
+        "members": [user_name],
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Added {} to group {}", user_name, group_name);
+
+    // 4. Attach policy to group
+    let set_policy_url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
+        env.url, policy_name, group_name
+    );
+    awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
+    info!("Attached policy {} to group {}", policy_name, group_name);
+
+    // 5. User with only group (no user policy) should be able to list buckets
+    let user_client = create_user_s3_client(&env, user_name, user_secret);
+    let list_result = user_client.list_buckets().send().await;
+    list_result.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> {
+        format!("User with only group membership should get group policies and list buckets: {}", e).into()
+    })?;
+    info!("User with only group successfully listed buckets (group policies applied)");
+
+    Ok(())
+}
+
+/// Test that after deleting a user who was the only member of a group, the group can be deleted
+/// (regression test for #2028.2: delete group uses backend membership, not stale cache).
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+#[ignore = "requires awscurl and spawns a real RustFS server"]
+async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let user_name = "solemember";
+    let user_secret = "solemembersecret";
+    let group_name = "soledeletegroup";
+
+    // 1. Create user
+    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
+    let user_body = serde_json::json!({
+        "secretKey": user_secret,
+        "status": "enabled"
+    });
+    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Created user {}", user_name);
+
+    // 2. Add user to group
+    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let add_member_body = serde_json::json!({
+        "group": group_name,
+        "members": [user_name],
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    info!("Added {} to group {}", user_name, group_name);
+
+    // 3. Delete the user (backend and cache update so group membership becomes empty)
+    let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
+    awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
+    info!("Deleted user {}", user_name);
+
+    // 4. Deleting the group should succeed (backend has empty members; no stale cache)
+    let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
+    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    info!("Deleted group {} after user was removed", group_name);
+
+    Ok(())
+}

crates/e2e_test/src/lib.rs

@@ -29,6 +29,10 @@ mod data_usage_test;
 #[cfg(test)]
 mod kms;
 
+// Regression test for issue #1797
+#[cfg(test)]
+mod list_objects_duplicates_test;
+
 // Quota tests
 #[cfg(test)]
 mod quota_test;
@@ -36,6 +40,10 @@ mod quota_test;
 #[cfg(test)]
 mod bucket_policy_check_test;
 
+// Regression tests for Issue #2036: anonymous access with PublicAccessBlock
+#[cfg(test)]
+mod anonymous_access_test;
+
 // Special characters in path test modules
 #[cfg(test)]
 mod special_chars_test;
@@ -54,6 +62,10 @@ mod policy;
 
 #[cfg(test)]
 mod compression_test;
+
+// Regression test for Issue #1878: DeleteMarkers not visible immediately after delete_objects
+#[cfg(test)]
+mod delete_objects_versioning_test;
 #[cfg(test)]
 mod protocols;
 
@@ -67,3 +79,7 @@ mod cluster_concurrency_test;
 // PutObject / MultipartUpload with checksum (Content-MD5, x-amz-checksum-*)
 #[cfg(test)]
 mod checksum_upload_test;
+
+// Group deletion tests
+#[cfg(test)]
+mod group_delete_test;

crates/e2e_test/src/list_objects_duplicates_test.rs

@@ -0,0 +1,135 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::primitives::ByteStream;
+    use serial_test::serial;
+    use tracing::info;
+
+    /// Helper function to create an S3 client for testing
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    /// Helper function to create a test bucket
+    async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let mut retries = 20;
+        loop {
+            match client.create_bucket().bucket(bucket).send().await {
+                Ok(_) => {
+                    info!("Bucket {} created successfully", bucket);
+                    return Ok(());
+                }
+                Err(e) => {
+                    // Ignore if bucket already exists
+                    if e.to_string().contains("BucketAlreadyOwnedByYou") || e.to_string().contains("BucketAlreadyExists") {
+                        info!("Bucket {} already exists", bucket);
+                        return Ok(());
+                    }
+                    if retries > 0 {
+                        retries -= 1;
+                        info!("Bucket creation failed, retrying... ({})", retries);
+                        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+                        continue;
+                    }
+                    return Err(Box::new(e));
+                }
+            }
+        }
+    }
+
+    /// Test ensuring that ListObjectsV2 returns unique CommonPrefixes even if "folder" objects exist.
+    ///
+    /// Bug Reference: Issue #1797
+    /// Veeam creates 0-byte objects ending in '/' (e.g. "folder/") to represent folders.
+    /// If "folder/file.txt" also exists, "folder/" is a CommonPrefix.
+    /// The bug was that "folder/" (the object) and "folder/" (derived prefix) were both added to CommonPrefixes
+    /// when delimiter was "/" because the deduplication check was explicitly skipped for "/" delimiter.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_unique_common_prefixes() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 should return unique CommonPrefixes");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-unique-prefixes";
+
+        // Create bucket
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        // 1. Create a file inside a folder
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("folder/file.txt")
+            .body(ByteStream::from_static(b"content"))
+            .send()
+            .await
+            .expect("Failed to create file inside folder");
+
+        // 2. Create the "folder" object itself (Veeam behavior)
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("folder/")
+            .body(ByteStream::from_static(b""))
+            .send()
+            .await
+            .expect("Failed to create folder object");
+
+        // 3. List with delimiter="/"
+        let result = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .delimiter("/")
+            .send()
+            .await
+            .expect("Failed to list objects");
+
+        // Verify prefixes
+        let prefixes = result.common_prefixes();
+        info!("CommonPrefixes: {:?}", prefixes);
+
+        // Should contain "folder/" exactly once
+        let folder_prefixes: Vec<_> = prefixes.iter().filter(|p| p.prefix() == Some("folder/")).collect();
+
+        assert_eq!(
+            folder_prefixes.len(),
+            1,
+            "Expected exactly 1 'folder/' prefix, found {}",
+            folder_prefixes.len()
+        );
+
+        // Verify that "folder/" is NOT returned as an object in Contents.
+        // For this regression test, we expect "folder/" to be represented only as a CommonPrefix
+        // (rolled up from "folder/file.txt" and the explicit "folder/" object), and to appear there
+        // exactly once. It must not appear in Contents at all.
+
+        // Ensure "folder/" is NOT in contents (Contents)
+        let folder_in_contents = result.contents().iter().any(|o| o.key() == Some("folder/"));
+        assert!(
+            !folder_in_contents,
+            "Expected 'folder/' to be rolled up into CommonPrefixes, but found it in Contents"
+        );
+
+        // Stop the RustFS server to ensure proper cleanup
+        env.stop_server();
+    }
+}

crates/e2e_test/src/protocols/ftps_core.rs

@@ -18,8 +18,7 @@ use crate::common::rustfs_binary_path;
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
 use rcgen::generate_simple_self_signed;
-use rustls::crypto::aws_lc_rs::default_provider;
-use rustls::{ClientConfig, RootCertStore};
+use rustls::{ClientConfig, RootCertStore, pki_types::CertificateDer, pki_types::pem::PemObject};
 use std::io::Cursor;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -74,8 +73,8 @@ pub async fn test_ftps_core_operations() -> Result<()> {
             .await
             .map_err(|e| anyhow::anyhow!("{}", e))?;
 
-        // Install the aws-lc-rs crypto provider
-        default_provider()
+        // Build ServerConfig with SNI support
+        rustls::crypto::aws_lc_rs::default_provider()
             .install_default()
             .map_err(|e| anyhow::anyhow!("Failed to install crypto provider: {:?}", e))?;
 
@@ -84,7 +83,7 @@ pub async fn test_ftps_core_operations() -> Result<()> {
         // Add the self-signed certificate to the trust store for e2e
         // Note: In a real environment, you'd use proper root certificates
         let cert_pem = default_cert.cert.pem();
-        let cert_der = rustls_pemfile::certs(&mut Cursor::new(cert_pem))
+        let cert_der = CertificateDer::pem_reader_iter(&mut Cursor::new(cert_pem))
             .collect::<Result<Vec<_>, _>>()
             .map_err(|e| anyhow::anyhow!("Failed to parse cert: {}", e))?;
 

crates/ecstore/Cargo.toml

@@ -111,6 +111,8 @@ google-cloud-storage = { workspace = true }
 google-cloud-auth = { workspace = true }
 aws-config = { workspace = true }
 faster-hex = { workspace = true }
+ratelimit = { workspace = true }
+aws-smithy-http-client.workspace = true
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

crates/ecstore/src/bucket/bandwidth/mod.rs

@@ -0,0 +1,16 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod monitor;
+pub mod reader;

crates/ecstore/src/bucket/bandwidth/monitor.rs

@@ -0,0 +1,321 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::bucket::bandwidth::reader::BucketOptions;
+use ratelimit::{Error as RatelimitError, Ratelimiter};
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex, RwLock};
+use std::time::Duration;
+use tracing::warn;
+
+#[derive(Clone)]
+pub struct BucketThrottle {
+    limiter: Arc<Mutex<Ratelimiter>>,
+    pub node_bandwidth_per_sec: i64,
+}
+
+impl BucketThrottle {
+    fn new(node_bandwidth_per_sec: i64) -> Result<Self, RatelimitError> {
+        let node_bandwidth_per_sec = node_bandwidth_per_sec.max(1);
+        let amount = node_bandwidth_per_sec as u64;
+        let limiter_inner = Ratelimiter::builder(amount, Duration::from_secs(1))
+            .max_tokens(amount)
+            .build()?;
+        Ok(Self {
+            limiter: Arc::new(Mutex::new(limiter_inner)),
+            node_bandwidth_per_sec,
+        })
+    }
+
+    pub fn burst(&self) -> u64 {
+        self.limiter.lock().unwrap_or_else(|e| e.into_inner()).max_tokens()
+    }
+
+    /// The ratelimit crate (0.10.0) does not provide a bulk token consumption API.
+    /// try_wait() first to consume 1 token AND trigger the internal refill
+    /// mechanism (tokens are only refilled during try_wait/wait calls).
+    /// directly adjust available tokens via set_available() to consume the remaining amount.
+    pub(crate) fn consume(&self, n: u64) -> (u64, f64, u64) {
+        let guard = self.limiter.lock().unwrap_or_else(|e| {
+            warn!("bucket throttle mutex poisoned, recovering");
+            e.into_inner()
+        });
+        if n == 0 {
+            return (0, guard.rate(), 0);
+        }
+        let mut consumed = 0u64;
+        if guard.try_wait().is_ok() {
+            consumed = 1;
+        }
+        let available = guard.available();
+        let to_consume = n - consumed;
+        let batch = to_consume.min(available);
+        if batch > 0 {
+            let _ = guard.set_available(available - batch);
+            consumed += batch;
+        }
+        let deficit = n.saturating_sub(consumed);
+        let rate = guard.rate();
+        (deficit, rate, consumed)
+    }
+}
+
+pub struct Monitor {
+    t_lock: RwLock<HashMap<BucketOptions, BucketThrottle>>,
+    pub node_count: u64,
+}
+
+impl Monitor {
+    pub fn new(num_nodes: u64) -> Arc<Self> {
+        let node_cnt = num_nodes.max(1);
+        Arc::new(Monitor {
+            t_lock: RwLock::new(HashMap::new()),
+            node_count: node_cnt,
+        })
+    }
+
+    pub fn delete_bucket(&self, bucket: &str) {
+        self.t_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .retain(|opts, _| opts.name != bucket);
+    }
+
+    pub fn delete_bucket_throttle(&self, bucket: &str, arn: &str) {
+        let opts = BucketOptions {
+            name: bucket.to_string(),
+            replication_arn: arn.to_string(),
+        };
+        self.t_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .remove(&opts);
+    }
+
+    pub fn throttle(&self, opts: &BucketOptions) -> Option<BucketThrottle> {
+        self.t_lock
+            .read()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor rwlock read poisoned, recovering");
+                e.into_inner()
+            })
+            .get(opts)
+            .cloned()
+    }
+
+    pub fn set_bandwidth_limit(&self, bucket: &str, arn: &str, limit: i64) {
+        if limit <= 0 {
+            warn!(
+                bucket = bucket,
+                arn = arn,
+                limit = limit,
+                "invalid bandwidth limit, must be positive; ignoring"
+            );
+            return;
+        }
+        let limit_bytes = limit / self.node_count as i64;
+        if limit_bytes == 0 && limit > 0 {
+            warn!(
+                bucket = bucket,
+                arn = arn,
+                limit = limit,
+                node_count = self.node_count,
+                "bandwidth limit too small for cluster size, per-node limit will clamp to 1 byte/s"
+            );
+        }
+        let opts = BucketOptions {
+            name: bucket.to_string(),
+            replication_arn: arn.to_string(),
+        };
+        let throttle = match BucketThrottle::new(limit_bytes) {
+            Ok(t) => t,
+            Err(e) => {
+                warn!(
+                    bucket = bucket,
+                    arn = arn,
+                    limit_bytes = limit_bytes,
+                    err = %e,
+                    "failed to build bandwidth throttle, throttling disabled for this target"
+                );
+                return;
+            }
+        };
+        self.t_lock
+            .write()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor rwlock write poisoned, recovering");
+                e.into_inner()
+            })
+            .insert(opts, throttle);
+    }
+
+    pub fn is_throttled(&self, bucket: &str, arn: &str) -> bool {
+        let opt = BucketOptions {
+            name: bucket.to_string(),
+            replication_arn: arn.to_string(),
+        };
+        self.t_lock
+            .read()
+            .unwrap_or_else(|e| {
+                warn!("bucket monitor rwlock read poisoned, recovering");
+                e.into_inner()
+            })
+            .contains_key(&opt)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_set_and_get_throttle_with_node_split() {
+        let monitor = Monitor::new(4);
+        monitor.set_bandwidth_limit("b1", "arn1", 400);
+
+        let throttle = monitor
+            .throttle(&BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            })
+            .expect("throttle should exist");
+
+        assert_eq!(throttle.node_bandwidth_per_sec, 100);
+        assert!(monitor.is_throttled("b1", "arn1"));
+    }
+
+    #[test]
+    fn test_delete_bucket_throttle() {
+        let monitor = Monitor::new(2);
+        monitor.set_bandwidth_limit("b1", "arn1", 200);
+        assert!(monitor.is_throttled("b1", "arn1"));
+
+        monitor.delete_bucket_throttle("b1", "arn1");
+        assert!(!monitor.is_throttled("b1", "arn1"));
+    }
+
+    #[test]
+    fn test_delete_bucket_removes_all_arns() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 100);
+        monitor.set_bandwidth_limit("b1", "arn2", 100);
+        monitor.set_bandwidth_limit("b2", "arn3", 100);
+
+        monitor.delete_bucket("b1");
+
+        assert!(!monitor.is_throttled("b1", "arn1"));
+        assert!(!monitor.is_throttled("b1", "arn2"));
+        assert!(monitor.is_throttled("b2", "arn3"));
+    }
+
+    #[test]
+    fn test_set_bandwidth_limit_ignores_non_positive() {
+        let monitor = Monitor::new(2);
+
+        monitor.set_bandwidth_limit("b1", "arn1", 0);
+        assert!(!monitor.is_throttled("b1", "arn1"));
+
+        monitor.set_bandwidth_limit("b1", "arn1", -10);
+        assert!(!monitor.is_throttled("b1", "arn1"));
+    }
+
+    #[test]
+    fn test_consume_returns_deficit_when_tokens_exhausted() {
+        let throttle = BucketThrottle::new(100).expect("test");
+
+        let (deficit, rate, _consumed) = throttle.consume(200);
+
+        assert!(deficit > 0);
+        assert!(rate > 0.0);
+    }
+
+    #[test]
+    fn test_consume_no_deficit_when_tokens_sufficient() {
+        let throttle = BucketThrottle::new(10000).expect("test");
+
+        std::thread::sleep(std::time::Duration::from_millis(1100));
+
+        let (deficit, _rate, _consumed) = throttle.consume(5000);
+        assert_eq!(deficit, 0);
+    }
+
+    #[test]
+    fn test_burst_equals_bandwidth() {
+        let throttle = BucketThrottle::new(500).expect("test");
+        assert_eq!(throttle.burst(), 500);
+    }
+
+    #[test]
+    fn test_concurrent_consume() {
+        let throttle = BucketThrottle::new(10000).expect("test");
+        std::thread::sleep(std::time::Duration::from_millis(1100));
+
+        let mut handles = vec![];
+        for _ in 0..10 {
+            let t = throttle.clone();
+            handles.push(std::thread::spawn(move || t.consume(100)));
+        }
+
+        let mut total_deficit = 0u64;
+        let mut total_consumed = 0u64;
+        for h in handles {
+            let (deficit, _rate, consumed) = h.join().unwrap();
+            total_consumed += consumed;
+            total_deficit += deficit;
+        }
+
+        assert_eq!(total_consumed + total_deficit, 1000);
+        assert!(total_consumed <= 10000);
+    }
+
+    #[test]
+    fn test_zero_bandwidth_clamped_to_one() {
+        let throttle = BucketThrottle::new(0).expect("test");
+        assert_eq!(throttle.burst(), 1);
+        assert_eq!(throttle.node_bandwidth_per_sec, 1);
+    }
+
+    #[test]
+    fn test_negative_bandwidth_clamped_to_one() {
+        let throttle = BucketThrottle::new(-100).expect("test");
+        assert_eq!(throttle.burst(), 1);
+        assert_eq!(throttle.node_bandwidth_per_sec, 1);
+    }
+
+    #[test]
+    fn test_update_bandwidth_limit_overrides_previous() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 1000);
+
+        let opts = BucketOptions {
+            name: "b1".to_string(),
+            replication_arn: "arn1".to_string(),
+        };
+        let t1 = monitor.throttle(&opts).expect("throttle should exist");
+        assert_eq!(t1.burst(), 1000);
+        assert_eq!(t1.node_bandwidth_per_sec, 1000);
+
+        monitor.set_bandwidth_limit("b1", "arn1", 500);
+
+        let t2 = monitor.throttle(&opts).expect("throttle should exist after update");
+        assert_eq!(t2.burst(), 500);
+        assert_eq!(t2.node_bandwidth_per_sec, 500);
+    }
+}

crates/ecstore/src/bucket/bandwidth/reader.rs

@@ -0,0 +1,336 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::bucket::bandwidth::monitor::Monitor;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+use tokio::io::{AsyncRead, ReadBuf};
+use tokio::time::Sleep;
+use tracing::{debug, warn};
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct BucketOptions {
+    pub name: String,
+    pub replication_arn: String,
+}
+
+pub struct MonitorReaderOptions {
+    pub bucket_options: BucketOptions,
+    pub header_size: usize,
+}
+
+struct WaitState {
+    sleep: Pin<Box<Sleep>>,
+}
+
+pub struct MonitoredReader<R> {
+    r: R,
+    m: Arc<Monitor>,
+    opts: MonitorReaderOptions,
+    wait_state: std::sync::Mutex<Option<WaitState>>,
+    temp_buf: Vec<u8>,
+}
+
+impl<R> MonitoredReader<R> {
+    pub fn new(m: Arc<Monitor>, r: R, opts: MonitorReaderOptions) -> Self {
+        let throttle = m.throttle(&opts.bucket_options);
+        debug!(
+            bucket = opts.bucket_options.name,
+            arn = opts.bucket_options.replication_arn,
+            throttle_active = throttle.is_some(),
+            limit_bps = throttle.as_ref().map(|t| t.node_bandwidth_per_sec).unwrap_or(0),
+            header_size = opts.header_size,
+            "MonitoredReader created"
+        );
+        MonitoredReader {
+            r,
+            m,
+            opts,
+            wait_state: std::sync::Mutex::new(None),
+            temp_buf: Vec::new(),
+        }
+    }
+}
+
+impl<R: AsyncRead + Unpin> AsyncRead for MonitoredReader<R> {
+    fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+        let this = self.get_mut();
+
+        {
+            let mut guard = this.wait_state.lock().unwrap_or_else(|e| {
+                warn!("MonitoredReader wait_state mutex poisoned, recovering");
+                e.into_inner()
+            });
+            if let Some(ref mut ws) = *guard {
+                match ws.sleep.as_mut().poll(cx) {
+                    Poll::Pending => return Poll::Pending,
+                    Poll::Ready(()) => {
+                        *guard = None;
+                        drop(guard);
+                    }
+                }
+            }
+        }
+
+        let throttle = match this.m.throttle(&this.opts.bucket_options) {
+            Some(t) => t,
+            None => return Pin::new(&mut this.r).poll_read(cx, buf),
+        };
+
+        let b = throttle.burst();
+        debug_assert!(b >= 1, "burst must be at least 1");
+        let (need, tokens) = calc_need_and_tokens(b, buf.remaining(), &mut this.opts.header_size);
+        let (deficit, rate, consumed) = throttle.consume(tokens);
+        let need = need.min(consumed as usize);
+
+        if deficit > 0 && rate > 0.0 {
+            let duration = std::time::Duration::from_secs_f64(deficit as f64 / rate);
+            debug!(
+                tokens = tokens,
+                deficit = deficit,
+                rate = rate,
+                sleep_ms = duration.as_millis() as u64,
+                "bandwidth throttle sleep"
+            );
+            let mut sleep = Box::pin(tokio::time::sleep(duration));
+            match sleep.as_mut().poll(cx) {
+                Poll::Pending => {
+                    *this.wait_state.lock().unwrap_or_else(|e| {
+                        warn!("MonitoredReader wait_state mutex poisoned, recovering");
+                        e.into_inner()
+                    }) = Some(WaitState { sleep });
+                    return Poll::Pending;
+                }
+                Poll::Ready(()) => {}
+            }
+        }
+
+        poll_limited_read(&mut this.r, cx, buf, need, &mut this.temp_buf)
+    }
+}
+
+fn calc_need_and_tokens(burst: u64, need_upper: usize, header_size: &mut usize) -> (usize, u64) {
+    let hdr = *header_size;
+    let mut need = need_upper;
+    let tokens: u64 = if hdr > 0 {
+        if (hdr as u64) < burst {
+            *header_size = 0;
+            need = ((burst - hdr as u64) as usize).min(need);
+            need as u64 + hdr as u64
+        } else {
+            *header_size -= burst as usize;
+            need = 0;
+            burst
+        }
+    } else {
+        need = need.min(burst as usize);
+        need as u64
+    };
+    (need, tokens)
+}
+
+fn poll_limited_read<R: AsyncRead + Unpin>(
+    r: &mut R,
+    cx: &mut Context<'_>,
+    buf: &mut ReadBuf<'_>,
+    limit: usize,
+    reusable_buf: &mut Vec<u8>,
+) -> Poll<std::io::Result<()>> {
+    let remaining = buf.remaining();
+    if limit == 0 || remaining == 0 {
+        return Poll::Ready(Ok(()));
+    }
+
+    if remaining <= limit {
+        return Pin::new(r).poll_read(cx, buf);
+    }
+
+    reusable_buf.resize(limit, 0);
+    let mut temp_buf = ReadBuf::new(&mut reusable_buf[..limit]);
+    match Pin::new(r).poll_read(cx, &mut temp_buf) {
+        Poll::Ready(Ok(())) => {
+            buf.put_slice(temp_buf.filled());
+            Poll::Ready(Ok(()))
+        }
+        other => other,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures_util::task::noop_waker_ref;
+    use std::io;
+    use std::pin::Pin;
+    use std::task::Context;
+    use tokio::io::AsyncReadExt;
+
+    #[derive(Default)]
+    struct TestAsyncReader {
+        data: Vec<u8>,
+        pos: usize,
+    }
+
+    impl TestAsyncReader {
+        fn new(data: &[u8]) -> Self {
+            Self {
+                data: data.to_vec(),
+                pos: 0,
+            }
+        }
+    }
+
+    impl AsyncRead for TestAsyncReader {
+        fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
+            if self.pos >= self.data.len() {
+                return Poll::Ready(Ok(()));
+            }
+
+            let remaining = self.data.len() - self.pos;
+            let n = remaining.min(buf.remaining());
+            buf.put_slice(&self.data[self.pos..self.pos + n]);
+            self.pos += n;
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_passthrough_when_throttle_absent() {
+        let monitor = Monitor::new(1);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 0,
+        };
+        let inner = TestAsyncReader::new(b"hello-world");
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out, b"hello-world");
+    }
+
+    #[test]
+    fn test_poll_limited_read_honors_limit() {
+        let mut inner = TestAsyncReader::new(b"abcdef");
+        let mut out = [0u8; 8];
+        let mut reusable = Vec::new();
+        let waker = noop_waker_ref();
+        let mut cx = Context::from_waker(waker);
+
+        let mut read_buf = ReadBuf::new(&mut out);
+        let first = poll_limited_read(&mut inner, &mut cx, &mut read_buf, 3, &mut reusable);
+        assert!(matches!(first, Poll::Ready(Ok(()))));
+        assert_eq!(read_buf.filled(), b"abc");
+
+        let second = poll_limited_read(&mut inner, &mut cx, &mut read_buf, 3, &mut reusable);
+        assert!(matches!(second, Poll::Ready(Ok(()))));
+        assert_eq!(read_buf.filled(), b"abcdef");
+    }
+
+    #[tokio::test]
+    async fn test_header_only_consumption_skips_io() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 10);
+
+        let data = vec![0xAAu8; 20];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 25,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let waker = noop_waker_ref();
+        let mut cx = Context::from_waker(waker);
+        let mut out = [0u8; 16];
+        let mut read_buf = ReadBuf::new(&mut out);
+
+        let poll = Pin::new(&mut reader).poll_read(&mut cx, &mut read_buf);
+        assert!(matches!(poll, Poll::Ready(Ok(())) | Poll::Pending));
+        assert!(read_buf.filled().is_empty());
+        assert_eq!(reader.opts.header_size, 15);
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_with_throttle_reads_all_data() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 1024);
+
+        let data = vec![0xABu8; 4096];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 0,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out.len(), 4096);
+        assert!(out.iter().all(|&b| b == 0xAB));
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_header_size_accounting() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 100);
+
+        let data = vec![0u8; 200];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 50,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out.len(), 200);
+        assert_eq!(reader.opts.header_size, 0);
+    }
+
+    #[tokio::test]
+    async fn test_monitored_reader_very_small_limit() {
+        let monitor = Monitor::new(1);
+        monitor.set_bandwidth_limit("b1", "arn1", 1);
+
+        let data = vec![0xFFu8; 10];
+        let inner = TestAsyncReader::new(&data);
+        let opts = MonitorReaderOptions {
+            bucket_options: BucketOptions {
+                name: "b1".to_string(),
+                replication_arn: "arn1".to_string(),
+            },
+            header_size: 0,
+        };
+        let mut reader = MonitoredReader::new(monitor, inner, opts);
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out.len(), 10);
+    }
+}

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -21,6 +21,7 @@ use crate::bucket::target::ARN;
 use crate::bucket::target::BucketTargetType;
 use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials};
 use crate::bucket::versioning_sys::BucketVersioningSys;
+use crate::global::get_global_bucket_monitor;
 use aws_credential_types::Credentials as SdkCredentials;
 use aws_sdk_s3::config::Region as SdkRegion;
 use aws_sdk_s3::error::ProvideErrorMetadata;
@@ -35,8 +36,10 @@ use aws_sdk_s3::types::{
 };
 use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput};
 use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus};
+use aws_smithy_http_client::{Builder as SmithyHttpClientBuilder, tls as smithy_tls};
 use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
 use reqwest::Client as HttpClient;
+use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
 use rustfs_filemeta::{ReplicationStatusType, ReplicationType};
 use rustfs_utils::http::{
     AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
@@ -49,6 +52,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::error::Error;
 use std::fmt;
+use std::path::Path;
 use std::str::FromStr as _;
 use std::sync::Arc;
 use std::sync::OnceLock;
@@ -639,12 +643,23 @@ impl BucketTargetSys {
             format!("http://{}", target.endpoint)
         };
 
-        let config = S3Config::builder()
+        let mut config_builder = S3Config::builder()
             .endpoint_url(endpoint.clone())
             .credentials_provider(SharedCredentialsProvider::new(creds))
             .region(SdkRegion::new(target.region.clone()))
-            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
-            .build();
+            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
+
+        if should_force_path_style(target) {
+            config_builder = config_builder.force_path_style(true);
+        }
+
+        if target.secure
+            && let Some(http_client) = build_aws_s3_http_client_from_tls_path().await
+        {
+            config_builder = config_builder.http_client(http_client);
+        }
+
+        let config = config_builder.build();
 
         Ok(TargetClient {
             endpoint,
@@ -667,9 +682,19 @@ impl BucketTargetSys {
         Ok(true)
     }
 
-    fn update_bandwidth_limit(&self, _bucket: &str, _arn: &str, _limit: i64) {
-        // Implementation for bandwidth limit update
-        // This would interact with the global bucket monitor
+    fn update_bandwidth_limit(&self, bucket: &str, arn: &str, limit: i64) {
+        if let Some(bucket_monitor) = get_global_bucket_monitor() {
+            if limit == 0 {
+                bucket_monitor.delete_bucket_throttle(bucket, arn);
+                return;
+            }
+            bucket_monitor.set_bandwidth_limit(bucket, arn, limit);
+        } else {
+            error!(
+                "Global bucket monitor uninitialized; skipping bandwidth limit update for bucket '{}' and ARN '{}'",
+                bucket, arn
+            );
+        }
     }
 
     pub async fn get_remote_target_client_by_arn(&self, _bucket: &str, arn: &str) -> Option<Arc<TargetClient>> {
@@ -691,6 +716,7 @@ impl BucketTargetSys {
         if let Some(existing_targets) = targets_map.remove(bucket) {
             for target in existing_targets {
                 arn_remotes_map.remove(&target.arn);
+                self.update_bandwidth_limit(bucket, &target.arn, 0);
             }
         }
 
@@ -777,6 +803,72 @@ impl BucketTargetSys {
     }
 }
 
+async fn build_aws_s3_http_client_from_tls_path() -> Option<aws_sdk_s3::config::SharedHttpClient> {
+    let tls_path = std::env::var("RUSTFS_TLS_PATH").ok()?;
+    if tls_path.is_empty() {
+        return None;
+    }
+
+    let tls_dir = Path::new(&tls_path);
+    let mut trust_store = smithy_tls::TrustStore::default();
+    let mut has_custom_certs = false;
+
+    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
+    match tokio::fs::read(&ca_path).await {
+        Ok(pem) => {
+            trust_store.add_pem_certificate(pem);
+            has_custom_certs = true;
+        }
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
+    }
+
+    if rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) {
+        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
+        match tokio::fs::read(&leaf_cert_path).await {
+            Ok(pem) => {
+                trust_store.add_pem_certificate(pem);
+                has_custom_certs = true;
+            }
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
+        }
+    }
+
+    if !has_custom_certs {
+        return None;
+    }
+
+    let tls_context = match smithy_tls::TlsContext::builder().with_trust_store(trust_store).build() {
+        Ok(ctx) => ctx,
+        Err(e) => {
+            warn!("failed to build AWS SDK TLS context for replication client: {}", e);
+            return None;
+        }
+    };
+
+    Some(
+        SmithyHttpClientBuilder::new()
+            .tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
+            .tls_context(tls_context)
+            .build_https(),
+    )
+}
+
+fn should_force_path_style(target: &BucketTarget) -> bool {
+    match target.path.trim().to_ascii_lowercase().as_str() {
+        // Explicit DNS/virtual-hosted-style requested by user.
+        "dns" | "off" | "false" => false,
+        // Explicit path-style or legacy boolean-like values.
+        "path" | "on" | "true" => true,
+        // `auto` and empty are defaulted to path-style for custom S3-compatible endpoints.
+        // RustFS/MinIO-style deployments typically do not configure virtual-hosted-style routing.
+        "auto" | "" => true,
+        // Unknown values: prefer compatibility with S3-compatible services.
+        _ => true,
+    }
+}
+
 // generate ARN that is unique to this target type
 fn generate_arn(t: &BucketTarget, depl_id: &str) -> String {
     let uuid = if depl_id.is_empty() {

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -34,7 +34,7 @@ use crate::global::GLOBAL_LocalNodeName;
 use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
 use crate::store::ECStore;
 use crate::store_api::StorageAPI;
-use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
+use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete};
 use crate::tier::warm_backend::WarmBackendGetOpts;
 use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
 use bytes::BytesMut;
@@ -65,7 +65,7 @@ use time::OffsetDateTime;
 use tokio::select;
 use tokio::sync::mpsc::{Receiver, Sender};
 use tokio::sync::{RwLock, mpsc};
-use tracing::{debug, error, info};
+use tracing::{debug, error, info, warn};
 use uuid::Uuid;
 use xxhash_rust::xxh64;
 
@@ -102,7 +102,7 @@ impl LifecycleSys {
     }
 
     pub fn trace(_oi: &ObjectInfo) -> TraceFn {
-        todo!();
+        Arc::new(|_oi, _ctx| Box::pin(async move {}))
     }
 }
 
@@ -395,7 +395,7 @@ impl ExpiryState {
                     }
                     else {
                         //info!("Invalid work type - {:?}", v);
-                        todo!();
+                        warn!("lifecycle worker received unsupported operation type");
                     }
                 }
             }
@@ -782,12 +782,13 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
         mod_time: oi.mod_time,
         ..Default::default()
     };
-    time_ilm(1);
-    api.transition_object(&oi.bucket, &oi.name, &opts).await
+    let result = api.transition_object(&oi.bucket, &oi.name, &opts).await;
+    time_ilm(1)();
+    result
 }
 
 pub fn audit_tier_actions(_api: ECStore, _tier: &str, _bytes: i64) -> TimeFn {
-    todo!();
+    Arc::new(|| Box::pin(async move {}))
 }
 
 pub async fn get_transitioned_object_reader(
@@ -1082,11 +1083,11 @@ pub async fn apply_expiry_on_transitioned_object(
     lc_event: &lifecycle::Event,
     src: &LcEventSrc,
 ) -> bool {
-    // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
+    let time_ilm = Metrics::time_ilm(lc_event.action);
     if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await {
         return false;
     }
-    // let _ = time_ilm(1);
+    time_ilm(1)();
 
     true
 }
@@ -1114,7 +1115,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
         opts.delete_prefix_object = true;
     }
 
-    // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
+    let time_ilm = Metrics::time_ilm(lc_event.action);
 
     //debug!("lc_event.action: {:?}", lc_event.action);
     //debug!("opts: {:?}", opts);
@@ -1152,11 +1153,11 @@ pub async fn apply_expiry_on_non_transitioned_objects(
     });
 
     if lc_event.action != lifecycle::IlmAction::NoneAction {
-        // let mut num_versions = 1_u64;
-        // if lc_event.action.delete_all() {
-        //     num_versions = oi.num_versions as u64;
-        // }
-        // let _ = time_ilm(num_versions);
+        let mut num_versions = 1_u64;
+        if lc_event.action.delete_all() {
+            num_versions = oi.num_versions as u64;
+        }
+        time_ilm(num_versions)();
     }
 
     true

crates/ecstore/src/bucket/lifecycle/lifecycle.rs

@@ -45,6 +45,10 @@ const _ERR_XML_NOT_WELL_FORMED: &str =
 const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
     "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an retention bucket";
 const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
+const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days must be greater than 0";
+const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
+const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters";
+const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
 
 pub use rustfs_common::metrics::IlmAction;
 
@@ -138,6 +142,7 @@ pub trait Lifecycle {
     async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error>;
     async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>>;
     async fn eval(&self, obj: &ObjectOpts) -> Event;
+    async fn predict_expiration(&self, obj: &ObjectOpts) -> Event;
     async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime, newer_noncurrent_versions: usize) -> Event;
     //fn set_prediction_headers(&self, w: http.ResponseWriter, obj: ObjectOpts);
     async fn noncurrent_versions_expiration_limit(self: Arc<Self>, obj: &ObjectOpts) -> Event;
@@ -232,6 +237,30 @@ impl Lifecycle for BucketLifecycleConfiguration {
         }
 
         for r in &self.rules {
+            if r.status != ExpirationStatus::from_static(ExpirationStatus::ENABLED)
+                && r.status != ExpirationStatus::from_static(ExpirationStatus::DISABLED)
+            {
+                return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_STATUS));
+            }
+
+            if let Some(expiration) = &r.expiration {
+                if let Some(expiration_date) = &expiration.date {
+                    let date = OffsetDateTime::from(expiration_date.clone());
+                    if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT));
+                    }
+                }
+                if let Some(days) = expiration.days {
+                    if days <= 0 {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS));
+                    }
+                }
+            }
+            if let Some(id) = &r.id {
+                if id.len() > 255 {
+                    return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG));
+                }
+            }
             r.validate()?;
             /*if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() {
                 if let Some(expiration) = r.expiration.as_ref() {
@@ -249,8 +278,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
             }
             let other_rules = &self.rules[i + 1..];
             for other_rule in other_rules {
-                if self.rules[i].id == other_rule.id {
-                    return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
+                if let (Some(id1), Some(id2)) = (&self.rules[i].id, &other_rule.id) {
+                    if id1 == id2 {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
+                    }
                 }
             }
         }
@@ -287,6 +318,61 @@ impl Lifecycle for BucketLifecycleConfiguration {
         self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await
     }
 
+    async fn predict_expiration(&self, obj: &ObjectOpts) -> Event {
+        let mod_time = match obj.mod_time {
+            Some(time) => {
+                if time.unix_timestamp() == 0 {
+                    return Event::default();
+                }
+                time
+            }
+            None => return Event::default(),
+        };
+
+        if obj.delete_marker || !(obj.is_latest || obj.version_id.is_none_or(|v| v.is_nil())) {
+            return Event::default();
+        }
+
+        let Some(lc_rules) = self.filter_rules(obj).await else {
+            return Event::default();
+        };
+
+        let mut event: Option<Event> = None;
+        for rule in lc_rules {
+            let Some(expiration) = rule.expiration else {
+                continue;
+            };
+            if expiration.expired_object_delete_marker.is_some_and(|v| v) {
+                continue;
+            }
+            let Some(due) = (match (&expiration.date, expiration.days) {
+                (Some(date), _) => Some(OffsetDateTime::from(date.clone())),
+                (None, Some(days)) => Some(expected_expiry_time(mod_time, days)),
+                _ => None,
+            }) else {
+                continue;
+            };
+
+            let predicted = Event {
+                action: IlmAction::DeleteAction,
+                rule_id: rule.id.clone().unwrap_or_default(),
+                due: Some(due),
+                noncurrent_days: 0,
+                newer_noncurrent_versions: 0,
+                storage_class: "".into(),
+            };
+            let predicted_due = predicted.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp();
+            let should_replace = event
+                .as_ref()
+                .is_none_or(|e| predicted_due < e.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp());
+            if should_replace {
+                event = Some(predicted);
+            }
+        }
+
+        event.unwrap_or_default()
+    }
+
     async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime, newer_noncurrent_versions: usize) -> Event {
         let mut events = Vec::<Event>::new();
         info!(
@@ -770,3 +856,222 @@ impl Default for TransitionOptions {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn validate_rejects_non_positive_expiration_days() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(0),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc
+            .validate(&ObjectLockConfiguration::default())
+            .await
+            .expect_err("expected validation error");
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS);
+    }
+
+    #[tokio::test]
+    async fn validate_accepts_positive_expiration_days() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        lc.validate(&ObjectLockConfiguration::default())
+            .await
+            .expect("expected validation to pass");
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_non_midnight_expiration_date() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    date: Some(time::OffsetDateTime::from_unix_timestamp(20_000_101).unwrap().into()),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT);
+    }
+
+    #[tokio::test]
+    async fn predict_expiration_selects_closest_expiry_for_put_object() {
+        let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        days: Some(30),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: Some("rule-days".to_string()),
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        date: Some((base_time + Duration::days(1)).into()),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: Some("rule-date".to_string()),
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+            ],
+        };
+        let opts = ObjectOpts {
+            name: "obj".to_string(),
+            mod_time: Some(base_time),
+            is_latest: true,
+            version_id: None,
+            ..Default::default()
+        };
+
+        let event = lc.predict_expiration(&opts).await;
+        let expected = base_time + Duration::days(1);
+
+        assert_eq!(event.action, IlmAction::DeleteAction);
+        assert_eq!(event.rule_id, "rule-date");
+        assert_eq!(event.due, Some(expected));
+    }
+
+    #[tokio::test]
+    async fn validate_accepts_multiple_rules_without_ids() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        days: Some(30),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: None,
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+                LifecycleRule {
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    expiration: Some(LifecycleExpiration {
+                        days: Some(31),
+                        ..Default::default()
+                    }),
+                    abort_incomplete_multipart_upload: None,
+                    filter: None,
+                    id: None,
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    transitions: None,
+                },
+            ],
+        };
+
+        lc.validate(&ObjectLockConfiguration::default())
+            .await
+            .expect("expected validation to pass");
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_rule_id_too_long() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: Some("a".repeat(256)),
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG);
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_invalid_status_case_sensitive() {
+        let lc = BucketLifecycleConfiguration {
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static("enabled"),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                filter: None,
+                id: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_RULE_STATUS);
+    }
+}

crates/ecstore/src/bucket/metadata.rs

@@ -26,7 +26,8 @@ use rmp_serde::Serializer as rmpSerializer;
 use rustfs_policy::policy::BucketPolicy;
 use s3s::dto::{
     BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration,
-    ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
+    PublicAccessBlockConfiguration, ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging,
+    VersioningConfiguration,
 };
 use serde::Serializer;
 use serde::{Deserialize, Serialize};
@@ -50,6 +51,8 @@ pub const BUCKET_VERSIONING_CONFIG: &str = "versioning.xml";
 pub const BUCKET_REPLICATION_CONFIG: &str = "replication.xml";
 pub const BUCKET_TARGETS_FILE: &str = "bucket-targets.json";
 pub const BUCKET_CORS_CONFIG: &str = "cors.xml";
+pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
+pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
 
 #[derive(Debug, Deserialize, Serialize, Clone)]
 #[serde(rename_all = "PascalCase", default)]
@@ -69,6 +72,8 @@ pub struct BucketMetadata {
     pub bucket_targets_config_json: Vec<u8>,
     pub bucket_targets_config_meta_json: Vec<u8>,
     pub cors_config_xml: Vec<u8>,
+    pub public_access_block_config_xml: Vec<u8>,
+    pub bucket_acl_config_json: Vec<u8>,
 
     pub policy_config_updated_at: OffsetDateTime,
     pub object_lock_config_updated_at: OffsetDateTime,
@@ -82,6 +87,8 @@ pub struct BucketMetadata {
     pub bucket_targets_config_updated_at: OffsetDateTime,
     pub bucket_targets_config_meta_updated_at: OffsetDateTime,
     pub cors_config_updated_at: OffsetDateTime,
+    pub public_access_block_config_updated_at: OffsetDateTime,
+    pub bucket_acl_config_updated_at: OffsetDateTime,
 
     #[serde(skip)]
     pub new_field_updated_at: OffsetDateTime,
@@ -110,6 +117,10 @@ pub struct BucketMetadata {
     pub bucket_target_config_meta: Option<HashMap<String, String>>,
     #[serde(skip)]
     pub cors_config: Option<CORSConfiguration>,
+    #[serde(skip)]
+    pub public_access_block_config: Option<PublicAccessBlockConfiguration>,
+    #[serde(skip)]
+    pub bucket_acl_config: Option<String>,
 }
 
 impl Default for BucketMetadata {
@@ -130,6 +141,8 @@ impl Default for BucketMetadata {
             bucket_targets_config_json: Default::default(),
             bucket_targets_config_meta_json: Default::default(),
             cors_config_xml: Default::default(),
+            public_access_block_config_xml: Default::default(),
+            bucket_acl_config_json: Default::default(),
             policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
@@ -142,6 +155,8 @@ impl Default for BucketMetadata {
             bucket_targets_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             bucket_targets_config_meta_updated_at: OffsetDateTime::UNIX_EPOCH,
             cors_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
             policy_config: Default::default(),
             notification_config: Default::default(),
@@ -155,6 +170,8 @@ impl Default for BucketMetadata {
             bucket_target_config: Default::default(),
             bucket_target_config_meta: Default::default(),
             cors_config: Default::default(),
+            public_access_block_config: Default::default(),
+            bucket_acl_config: Default::default(),
         }
     }
 }
@@ -254,6 +271,12 @@ impl BucketMetadata {
         if self.bucket_targets_config_meta_updated_at == OffsetDateTime::UNIX_EPOCH {
             self.bucket_targets_config_meta_updated_at = self.created
         }
+        if self.public_access_block_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.public_access_block_config_updated_at = self.created
+        }
+        if self.bucket_acl_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.bucket_acl_config_updated_at = self.created
+        }
     }
 
     pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
@@ -307,6 +330,14 @@ impl BucketMetadata {
                 self.cors_config_xml = data;
                 self.cors_config_updated_at = updated;
             }
+            BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => {
+                self.public_access_block_config_xml = data;
+                self.public_access_block_config_updated_at = updated;
+            }
+            BUCKET_ACL_CONFIG => {
+                self.bucket_acl_config_json = data;
+                self.bucket_acl_config_updated_at = updated;
+            }
             _ => return Err(Error::other(format!("config file not found : {config_file}"))),
         }
 
@@ -380,6 +411,15 @@ impl BucketMetadata {
         if !self.cors_config_xml.is_empty() {
             self.cors_config = Some(deserialize::<CORSConfiguration>(&self.cors_config_xml)?);
         }
+        if !self.public_access_block_config_xml.is_empty() {
+            self.public_access_block_config =
+                Some(deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)?);
+        }
+        if !self.bucket_acl_config_json.is_empty() {
+            let acl = String::from_utf8(self.bucket_acl_config_json.clone())
+                .map_err(|e| Error::other(format!("invalid UTF-8 in bucket ACL: {}", e)))?;
+            self.bucket_acl_config = Some(acl);
+        }
 
         Ok(())
     }
@@ -530,6 +570,15 @@ mod test {
         bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec();
         bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc();
 
+        // Add public access block configuration
+        let public_access_block_xml = r#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#;
+        bm.public_access_block_config_xml = public_access_block_xml.as_bytes().to_vec();
+        bm.public_access_block_config_updated_at = OffsetDateTime::now_utc();
+
+        let bucket_acl = r#"{"owner":{"id":"rustfsadmin","display_name":"RustFS Tester"},"grants":[{"grantee":{"grantee_type":"CanonicalUser","id":"rustfsadmin","display_name":"RustFS Tester","uri":null,"email_address":null},"permission":"FULL_CONTROL"}]}"#;
+        bm.bucket_acl_config_json = bucket_acl.as_bytes().to_vec();
+        bm.bucket_acl_config_updated_at = OffsetDateTime::now_utc();
+
         // Test serialization
         let buf = bm.marshal_msg().unwrap();
         assert!(!buf.is_empty(), "Serialized buffer should not be empty");
@@ -549,6 +598,8 @@ mod test {
         assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml);
         assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml);
         assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json);
+        assert_eq!(bm.public_access_block_config_xml, deserialized_bm.public_access_block_config_xml);
+        assert_eq!(bm.bucket_acl_config_json, deserialized_bm.bucket_acl_config_json);
         assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml);
         assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml);
         assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml);

crates/ecstore/src/bucket/metadata_sys.rs

@@ -15,13 +15,13 @@
 use super::metadata::{BucketMetadata, load_bucket_metadata};
 use super::quota::BucketQuota;
 use super::target::BucketTargets;
-use crate::StorageAPI as _;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_parse};
 use crate::bucket::utils::{deserialize, is_meta_bucketname};
 use crate::error::{Error, Result, is_err_bucket_not_found};
 use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_layer_fn};
 use crate::store::ECStore;
+use crate::store_api::HealOperations as _;
 use futures::future::join_all;
 use lazy_static::lazy_static;
 use rustfs_common::heal_channel::HealOpts;
@@ -29,7 +29,7 @@ use rustfs_policy::policy::BucketPolicy;
 use s3s::dto::ReplicationConfiguration;
 use s3s::dto::{
     BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration,
-    ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
+    PublicAccessBlockConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
 };
 use std::collections::HashSet;
 use std::sync::OnceLock;
@@ -53,6 +53,10 @@ pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
     GLOBAL_BucketMetadataSys.set(sys).unwrap();
 }
 
+pub fn get_global_bucket_metadata_sys() -> Option<Arc<RwLock<BucketMetadataSys>>> {
+    GLOBAL_BucketMetadataSys.get().cloned()
+}
+
 // panic if not init
 pub(super) fn get_bucket_metadata_sys() -> Result<Arc<RwLock<BucketMetadataSys>>> {
     if let Some(sys) = GLOBAL_BucketMetadataSys.get() {
@@ -105,6 +109,13 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
     bucket_meta_sys.get_bucket_policy_raw(bucket).await
 }
 
+pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_bucket_acl_config(bucket).await
+}
+
 pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -133,6 +144,13 @@ pub async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime
     bucket_meta_sys.get_tagging_config(bucket).await
 }
 
+pub async fn get_public_access_block_config(bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_public_access_block_config(bucket).await
+}
+
 pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -478,6 +496,16 @@ impl BucketMetadataSys {
         }
     }
 
+    pub async fn get_bucket_acl_config(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.bucket_acl_config {
+            Ok((config.clone(), bm.bucket_acl_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
     pub async fn get_tagging_config(&self, bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
@@ -488,6 +516,16 @@ impl BucketMetadataSys {
         }
     }
 
+    pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.public_access_block_config {
+            Ok((config.clone(), bm.public_access_block_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
     pub async fn get_object_lock_config(&self, bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 

crates/ecstore/src/bucket/mod.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod bandwidth;
 pub mod bucket_target_sys;
 pub mod error;
 pub mod lifecycle;

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -977,6 +977,10 @@ pub async fn init_background_replication<S: StorageAPI>(storage: Arc<S>) {
     assert!(GLOBAL_REPLICATION_POOL.get().is_some());
 }
 
+pub fn get_global_replication_pool() -> Option<Arc<DynReplicationPool>> {
+    GLOBAL_REPLICATION_POOL.get().cloned()
+}
+
 pub async fn schedule_replication<S: StorageAPI>(oi: ObjectInfo, o: Arc<S>, dsc: ReplicateDecision, op_type: ReplicationType) {
     let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
     let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::bucket::bandwidth::reader::{BucketOptions, MonitorReaderOptions, MonitoredReader};
 use crate::bucket::bucket_target_sys::{
     AdvancedPutOptions, BucketTargetSys, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
 };
@@ -28,16 +29,21 @@ use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_fo
 use crate::event::name::EventName;
 use crate::event_notification::{EventArgs, send_event};
 use crate::global::GLOBAL_LocalNodeName;
+use crate::global::get_global_bucket_monitor;
 use crate::set_disk::get_lock_acquire_timeout;
-use crate::store_api::{DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
+use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
 use crate::{StorageAPI, new_object_layer_fn};
 use aws_sdk_s3::error::SdkError;
 use aws_sdk_s3::operation::head_object::HeadObjectOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
+use aws_smithy_types::body::SdkBody;
 use byteorder::ByteOrder;
 use futures::future::join_all;
+use futures::stream::StreamExt;
 use http::HeaderMap;
+use http_body::Frame;
+use http_body_util::StreamBody;
 use regex::Regex;
 use rustfs_filemeta::{
     MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATION_RESET, ReplicateDecision, ReplicateObjectInfo,
@@ -61,10 +67,11 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
-use tokio::io::{AsyncRead, AsyncReadExt};
+use tokio::io::AsyncRead;
 use tokio::sync::RwLock;
 use tokio::task::JoinSet;
 use tokio::time::Duration as TokioDuration;
+use tokio_util::io::ReaderStream;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};
 
@@ -74,6 +81,8 @@ const RESYNC_META_FORMAT: u16 = 1;
 const RESYNC_META_VERSION: u16 = 1;
 const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60);
 
+static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new();
+
 #[derive(Debug, Clone, Default)]
 pub struct ResyncOpts {
     pub bucket: String,
@@ -1193,6 +1202,11 @@ pub async fn must_replicate(bucket: &str, object: &str, mopts: MustReplicateOpti
 }
 
 pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo, storage: Arc<S>) {
+    if dobj.delete_object.force_delete {
+        replicate_force_delete_to_targets(&dobj, storage).await;
+        return;
+    }
+
     let bucket = dobj.bucket.clone();
     let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
         Some(version_id.to_owned())
@@ -1481,6 +1495,189 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
     }
 }
 
+async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
+    let bucket = &dobj.bucket;
+    let object_name = &dobj.delete_object.object_name;
+
+    let rcfg = match get_replication_config(bucket).await {
+        Ok(Some(config)) => config,
+        Ok(None) => {
+            warn!("replicate force-delete: no replication config for bucket:{}", bucket);
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object_name.clone(),
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+        Err(err) => {
+            warn!("replicate force-delete: replication config error bucket:{} error:{}", bucket, err);
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object_name.clone(),
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };
+
+    let ns_lock = match storage
+        .new_ns_lock(bucket, format!("/[replicate]/{}", object_name).as_str())
+        .await
+    {
+        Ok(ns_lock) => ns_lock,
+        Err(e) => {
+            warn!(
+                "replicate force-delete: failed to get ns lock bucket:{} object:{} error:{}",
+                bucket, object_name, e
+            );
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object_name.clone(),
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };
+
+    let _lock_guard = match ns_lock.get_write_lock(get_lock_acquire_timeout()).await {
+        Ok(guard) => guard,
+        Err(e) => {
+            warn!(
+                "replicate force-delete: failed to get write lock bucket:{} object:{} error:{}",
+                bucket, object_name, e
+            );
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object_name.clone(),
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };
+
+    let tgt_arns = if !dobj.target_arn.is_empty() {
+        vec![dobj.target_arn.clone()]
+    } else {
+        rcfg.filter_target_arns(&ObjectOpts {
+            name: object_name.clone(),
+            ..Default::default()
+        })
+    };
+
+    let mut join_set = JoinSet::new();
+
+    for arn in tgt_arns {
+        let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &arn).await else {
+            warn!("replicate force-delete: failed to get target client bucket:{} arn:{}", bucket, arn);
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object_name.clone(),
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            continue;
+        };
+
+        let bucket = bucket.clone();
+        let object_name = object_name.clone();
+
+        join_set.spawn(async move {
+            if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
+                error!("replicate force-delete: target offline bucket:{} arn:{}", bucket, tgt_client.arn);
+                send_event(EventArgs {
+                    event_name: EventName::ObjectReplicationFailed.as_ref().to_string(),
+                    bucket_name: bucket.clone(),
+                    object: ObjectInfo {
+                        bucket: bucket.clone(),
+                        name: object_name.clone(),
+                        ..Default::default()
+                    },
+                    user_agent: "Internal: [Replication]".to_string(),
+                    host: GLOBAL_LocalNodeName.to_string(),
+                    ..Default::default()
+                });
+                return;
+            }
+
+            if let Err(e) = tgt_client
+                .remove_object(
+                    &tgt_client.bucket,
+                    &object_name,
+                    None,
+                    RemoveObjectOptions {
+                        force_delete: true,
+                        governance_bypass: false,
+                        replication_delete_marker: false,
+                        replication_mtime: None,
+                        replication_status: ReplicationStatusType::Replica,
+                        replication_request: true,
+                        replication_validity_check: false,
+                    },
+                )
+                .await
+            {
+                error!(
+                    "replicate force-delete failed bucket:{} object:{} arn:{} error:{}",
+                    bucket, object_name, tgt_client.arn, e
+                );
+                send_event(EventArgs {
+                    event_name: EventName::ObjectReplicationFailed.as_ref().to_string(),
+                    bucket_name: bucket.clone(),
+                    object: ObjectInfo {
+                        bucket: bucket.clone(),
+                        name: object_name.clone(),
+                        ..Default::default()
+                    },
+                    user_agent: "Internal: [Replication]".to_string(),
+                    host: GLOBAL_LocalNodeName.to_string(),
+                    ..Default::default()
+                });
+            }
+        });
+    }
+
+    while let Some(result) = join_set.join_next().await {
+        if let Err(e) = result {
+            error!("replicate force-delete task panicked: {}", e);
+        }
+    }
+}
+
 async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
     let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
         version_id.to_owned()
@@ -1768,20 +1965,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
         let versioned = BucketVersioningSys::prefix_enabled(&bucket, &object).await;
         let version_suspended = BucketVersioningSys::prefix_suspended(&bucket, &object).await;
 
+        let obj_opts = ObjectOptions {
+            version_id: self.version_id.map(|v| v.to_string()),
+            version_suspended,
+            versioned,
+            replication_request: true,
+            ..Default::default()
+        };
+
         let mut gr = match storage
-            .get_object_reader(
-                &bucket,
-                &object,
-                None,
-                HeaderMap::new(),
-                &ObjectOptions {
-                    version_id: self.version_id.map(|v| v.to_string()),
-                    version_suspended,
-                    versioned,
-                    replication_request: true,
-                    ..Default::default()
-                },
-            )
+            .get_object_reader(&bucket, &object, None, HeaderMap::new(), &obj_opts)
             .await
         {
             Ok(gr) => gr,
@@ -1858,6 +2051,10 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                         warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
                         return rinfo;
                     }
+                } else if e.raw_response().is_some_and(|resp| resp.status().as_u16() == 404) {
+                    // Some HEAD Object 404 responses are surfaced by the AWS SDK as `response error`
+                    // instead of `service error (NotFound)`. Treat raw HTTP 404 as object-not-found
+                    // so replication can proceed with PUT.
                 } else {
                     rinfo.error = Some(e.to_string());
                     warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
@@ -1890,34 +2087,26 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             }
         };
 
-        // TODO:bandwidth
-
         if let Some(err) = if is_multipart {
-            replicate_object_with_multipart(tgt_client.clone(), &tgt_client.bucket, &object, gr.stream, &object_info, put_opts)
-                .await
-                .err()
+            drop(gr);
+            replicate_object_with_multipart(MultipartReplicationContext {
+                storage: storage.clone(),
+                cli: tgt_client.clone(),
+                src_bucket: &bucket,
+                dst_bucket: &tgt_client.bucket,
+                object: &object,
+                object_info: &object_info,
+                obj_opts: &obj_opts,
+                arn: &rinfo.arn,
+                put_opts,
+            })
+            .await
+            .err()
         } else {
-            // TODO: use stream
-            let body = match gr.read_all().await {
-                Ok(body) => body,
-                Err(e) => {
-                    rinfo.replication_status = ReplicationStatusType::Failed;
-                    rinfo.error = Some(e.to_string());
-                    warn!("failed to read object for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
-                    send_event(EventArgs {
-                        event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
-                        bucket_name: bucket.clone(),
-                        object: object_info.clone(),
-                        host: GLOBAL_LocalNodeName.to_string(),
-                        user_agent: "Internal: [Replication]".to_string(),
-                        ..Default::default()
-                    });
-                    return rinfo;
-                }
-            };
-            let reader = ByteStream::from(body);
+            gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
+            let byte_stream = async_read_to_bytestream(gr.stream);
             tgt_client
-                .put_object(&tgt_client.bucket, &object, size, reader, &put_opts)
+                .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                 .await
                 .map_err(|e| std::io::Error::other(e.to_string()))
                 .err()
@@ -1973,20 +2162,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
         let versioned = BucketVersioningSys::prefix_enabled(&bucket, &object).await;
         let version_suspended = BucketVersioningSys::prefix_suspended(&bucket, &object).await;
 
+        let obj_opts = ObjectOptions {
+            version_id: self.version_id.map(|v| v.to_string()),
+            version_suspended,
+            versioned,
+            replication_request: true,
+            ..Default::default()
+        };
+
         let mut gr = match storage
-            .get_object_reader(
-                &bucket,
-                &object,
-                None,
-                HeaderMap::new(),
-                &ObjectOptions {
-                    version_id: self.version_id.map(|v| v.to_string()),
-                    version_suspended,
-                    versioned,
-                    replication_request: true,
-                    ..Default::default()
-                },
-            )
+            .get_object_reader(&bucket, &object, None, HeaderMap::new(), &obj_opts)
             .await
         {
             Ok(gr) => gr,
@@ -2182,39 +2367,27 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                     return rinfo;
                 }
             };
+
             if let Some(err) = if is_multipart {
-                replicate_object_with_multipart(
-                    tgt_client.clone(),
-                    &tgt_client.bucket,
-                    &object,
-                    gr.stream,
-                    &object_info,
+                drop(gr);
+                replicate_object_with_multipart(MultipartReplicationContext {
+                    storage: storage.clone(),
+                    cli: tgt_client.clone(),
+                    src_bucket: &bucket,
+                    dst_bucket: &tgt_client.bucket,
+                    object: &object,
+                    object_info: &object_info,
+                    obj_opts: &obj_opts,
+                    arn: &rinfo.arn,
                     put_opts,
-                )
+                })
                 .await
                 .err()
             } else {
-                let body = match gr.read_all().await {
-                    Ok(body) => body,
-                    Err(e) => {
-                        rinfo.replication_status = ReplicationStatusType::Failed;
-                        rinfo.error = Some(e.to_string());
-                        warn!("failed to read object for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
-                        send_event(EventArgs {
-                            event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
-                            bucket_name: bucket.clone(),
-                            object: object_info,
-                            host: GLOBAL_LocalNodeName.to_string(),
-                            user_agent: "Internal: [Replication]".to_string(),
-                            ..Default::default()
-                        });
-                        rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
-                        return rinfo;
-                    }
-                };
-                let reader = ByteStream::from(body);
+                gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
+                let byte_stream = async_read_to_bytestream(gr.stream);
                 tgt_client
-                    .put_object(&tgt_client.bucket, &object, size, reader, &put_opts)
+                    .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                     .await
                     .map_err(|e| std::io::Error::other(e.to_string()))
                     .err()
@@ -2268,6 +2441,63 @@ static STANDARD_HEADERS: &[&str] = &[
     headers::AMZ_SERVER_SIDE_ENCRYPTION,
 ];
 
+fn calc_put_object_header_size(put_opts: &PutObjectOptions) -> usize {
+    let mut header_size: usize = 0;
+    for (key, value) in put_opts.header().iter() {
+        header_size += key.as_str().len();
+        header_size += value.as_bytes().len();
+        // Account for HTTP header formatting: ": " (2 bytes) and "\r\n" (2 bytes)
+        header_size += 4;
+    }
+    header_size
+}
+
+fn wrap_with_bandwidth_monitor_with_header(
+    stream: Box<dyn AsyncRead + Unpin + Send + Sync>,
+    bucket: &str,
+    arn: &str,
+    header_size: usize,
+) -> Box<dyn AsyncRead + Unpin + Send + Sync> {
+    if let Some(monitor) = get_global_bucket_monitor() {
+        Box::new(MonitoredReader::new(
+            monitor,
+            stream,
+            MonitorReaderOptions {
+                bucket_options: BucketOptions {
+                    name: bucket.to_string(),
+                    replication_arn: arn.to_string(),
+                },
+                header_size,
+            },
+        ))
+    } else {
+        WARNED_MONITOR_UNINIT.call_once(|| {
+            warn!(
+                "Global bucket monitor uninitialized; proceeding with unthrottled replication (bandwidth limits will be ignored)"
+            )
+        });
+        stream
+    }
+}
+
+fn wrap_with_bandwidth_monitor(
+    stream: Box<dyn AsyncRead + Unpin + Send + Sync>,
+    put_opts: &PutObjectOptions,
+    bucket: &str,
+    arn: &str,
+) -> Box<dyn AsyncRead + Unpin + Send + Sync> {
+    let header_size = calc_put_object_header_size(put_opts);
+    wrap_with_bandwidth_monitor_with_header(stream, bucket, arn, header_size)
+}
+
+fn async_read_to_bytestream(reader: impl AsyncRead + Send + Sync + Unpin + 'static) -> ByteStream {
+    // Non-retryable: SDK-level retries are not supported for streaming bodies.
+    // Replication-level retry handles failures at a higher layer.
+    let stream = ReaderStream::new(reader);
+    let body = StreamBody::new(stream.map(|r| r.map(Frame::data)));
+    ByteStream::new(SdkBody::from_body_1_x(body))
+}
+
 fn is_standard_header(k: &str) -> bool {
     STANDARD_HEADERS.iter().any(|h| h.eq_ignore_ascii_case(k))
 }
@@ -2491,17 +2721,56 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
     Ok((put_op, is_multipart))
 }
 
-async fn replicate_object_with_multipart(
+fn part_range_spec_from_actual_size(offset: i64, part_size: i64) -> std::io::Result<(HTTPRangeSpec, i64)> {
+    if offset < 0 {
+        return Err(std::io::Error::other("invalid part offset"));
+    }
+    if part_size <= 0 {
+        return Err(std::io::Error::other(format!("invalid part size {part_size}")));
+    }
+    let end = offset
+        .checked_add(part_size - 1)
+        .ok_or_else(|| std::io::Error::other("part range overflow"))?;
+    let next_offset = end
+        .checked_add(1)
+        .ok_or_else(|| std::io::Error::other("part offset overflow"))?;
+    Ok((
+        HTTPRangeSpec {
+            is_suffix_length: false,
+            start: offset,
+            end,
+        },
+        next_offset,
+    ))
+}
+
+struct MultipartReplicationContext<'a, S: StorageAPI> {
+    storage: Arc<S>,
     cli: Arc<TargetClient>,
-    bucket: &str,
-    object: &str,
-    reader: Box<dyn AsyncRead + Unpin + Send + Sync>,
-    object_info: &ObjectInfo,
-    opts: PutObjectOptions,
-) -> std::io::Result<()> {
+    src_bucket: &'a str,
+    dst_bucket: &'a str,
+    object: &'a str,
+    object_info: &'a ObjectInfo,
+    obj_opts: &'a ObjectOptions,
+    arn: &'a str,
+    put_opts: PutObjectOptions,
+}
+
+async fn replicate_object_with_multipart<S: StorageAPI>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
+    let MultipartReplicationContext {
+        storage,
+        cli,
+        src_bucket,
+        dst_bucket,
+        object,
+        object_info,
+        obj_opts,
+        arn,
+        put_opts,
+    } = ctx;
     let mut attempts = 1;
     let upload_id = loop {
-        match cli.create_multipart_upload(bucket, object, &opts).await {
+        match cli.create_multipart_upload(dst_bucket, object, &put_opts).await {
             Ok(id) => {
                 break id;
             }
@@ -2520,19 +2789,30 @@ async fn replicate_object_with_multipart(
 
     let mut uploaded_parts: Vec<CompletedPart> = Vec::new();
 
-    let mut reader = reader;
+    let mut header_size = calc_put_object_header_size(&put_opts);
+    let mut offset: i64 = 0;
     for part_info in object_info.parts.iter() {
-        let mut chunk = vec![0u8; part_info.actual_size as usize];
-        AsyncReadExt::read_exact(&mut *reader, &mut chunk).await?;
+        let part_size = part_info.actual_size;
+        let (range_spec, next_offset) = part_range_spec_from_actual_size(offset, part_size)?;
+        offset = next_offset;
+
+        let part_reader = storage
+            .get_object_reader(src_bucket, object, Some(range_spec), HeaderMap::new(), obj_opts)
+            .await
+            .map_err(|e| std::io::Error::other(e.to_string()))?;
+
+        let part_stream = wrap_with_bandwidth_monitor_with_header(part_reader.stream, src_bucket, arn, header_size);
+        header_size = 0;
+        let byte_stream = async_read_to_bytestream(part_stream);
 
         let object_part = cli
             .put_object_part(
-                bucket,
+                dst_bucket,
                 object,
                 &upload_id,
                 part_info.number as i32,
-                part_info.actual_size,
-                ByteStream::from(chunk),
+                part_size,
+                byte_stream,
                 &PutObjectPartOptions { ..Default::default() },
             )
             .await
@@ -2560,7 +2840,7 @@ async fn replicate_object_with_multipart(
     );
 
     cli.complete_multipart_upload(
-        bucket,
+        dst_bucket,
         object,
         &upload_id,
         uploaded_parts,
@@ -2681,3 +2961,22 @@ fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: Rep
 
     ReplicationAction::None
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_part_range_spec_from_actual_size() {
+        let (rs, next) = part_range_spec_from_actual_size(0, 10).unwrap();
+        assert_eq!(rs.start, 0);
+        assert_eq!(rs.end, 9);
+        assert_eq!(next, 10);
+    }
+
+    #[test]
+    fn test_part_range_spec_rejects_non_positive() {
+        assert!(part_range_spec_from_actual_size(0, 0).is_err());
+        assert!(part_range_spec_from_actual_size(0, -1).is_err());
+    }
+}

crates/ecstore/src/bucket/utils.rs

@@ -193,10 +193,6 @@ pub fn is_valid_object_name(object: &str) -> bool {
         return false;
     }
 
-    if object.ends_with(SLASH_SEPARATOR) {
-        return false;
-    }
-
     is_valid_object_prefix(object)
 }
 
@@ -379,10 +375,10 @@ mod tests {
         // Invalid cases - empty string
         assert!(!is_valid_object_name(""));
 
-        // Invalid cases - ends with slash (object names cannot end with slash)
-        assert!(!is_valid_object_name("object/"));
-        assert!(!is_valid_object_name("path/to/file/"));
-        assert!(!is_valid_object_name("ends/with/slash/"));
+        // Valid cases - trailing slash is allowed (empty object with trailing slash)
+        assert!(is_valid_object_name("object/"));
+        assert!(is_valid_object_name("path/to/file/"));
+        assert!(is_valid_object_name("ends/with/slash/"));
 
         // Invalid cases - bad path components (inherited from is_valid_object_prefix)
         assert!(!is_valid_object_name("."));

crates/ecstore/src/client/api_get_object.rs

@@ -41,7 +41,11 @@ use tokio_util::io::ReaderStream;
 
 impl TransitionClient {
     pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
-        todo!();
+        let _ = opts;
+        Err(std::io::Error::new(
+            IoErrorKind::Unsupported,
+            format!("get_object is not implemented for {bucket_name}/{object_name}"),
+        ))
     }
 
     pub async fn get_object_inner(
@@ -131,7 +135,18 @@ impl Object {
     }
 
     fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
-        todo!()
+        let _ = request.did_offset_change;
+        let _ = request.offset;
+        let _ = request.is_first_req;
+        let _ = request.is_read_at;
+        let _ = request.setting_object_info;
+        let _ = request.is_read_op;
+        let _ = request.been_read;
+        let _ = request.buffer.len();
+        Err(std::io::Error::new(
+            IoErrorKind::Unsupported,
+            "read-path for Object in api_get_object is not implemented",
+        ))
     }
 
     fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {

crates/ecstore/src/client/api_list.rs

@@ -34,10 +34,21 @@ use hyper::body::Bytes;
 use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
 use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
 use std::collections::HashMap;
+use std::io::ErrorKind;
 
 impl TransitionClient {
     pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> {
-        todo!();
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: "The list_buckets API is not implemented in this build.".to_string(),
+                },
+                request_id: "".to_string(),
+            },
+        ))
     }
 
     pub async fn list_objects_v2_query(
@@ -237,7 +248,17 @@ impl TransitionClient {
         }
 
         Ok(listObjectVersionsOutput)*/
-        todo!();
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!("list_object_versions_query is not implemented for bucket {bucket_name}"),
+                },
+                request_id: "".to_string(),
+            },
+        ))
     }
 
     pub fn list_objects_query(
@@ -249,7 +270,17 @@ impl TransitionClient {
         max_keys: i64,
         headers: HeaderMap,
     ) -> Result<ListBucketResult, std::io::Error> {
-        todo!();
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!("list_objects_query is not implemented for bucket {bucket_name}"),
+                },
+                request_id: "".to_string(),
+            },
+        ))
     }
 
     pub fn list_multipart_uploads_query(
@@ -261,7 +292,17 @@ impl TransitionClient {
         delimiter: &str,
         max_uploads: i64,
     ) -> Result<ListMultipartUploadsResult, std::io::Error> {
-        todo!();
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!("list_multipart_uploads_query is not implemented for bucket {bucket_name}"),
+                },
+                request_id: "".to_string(),
+            },
+        ))
     }
 
     pub fn list_object_parts(
@@ -270,11 +311,33 @@ impl TransitionClient {
         object_name: &str,
         upload_id: &str,
     ) -> Result<HashMap<i64, ObjectPart>, std::io::Error> {
-        todo!();
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!(
+                        "list_object_parts is not implemented for bucket {bucket_name}, object {object_name}, upload_id {upload_id}"
+                    ),
+                },
+                request_id: "".to_string(),
+            },
+        ))
     }
 
     pub fn find_upload_ids(&self, bucket_name: &str, object_name: &str) -> Result<Vec<String>, std::io::Error> {
-        todo!();
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!("find_upload_ids is not implemented for bucket {bucket_name}, object {object_name}"),
+                },
+                request_id: "".to_string(),
+            },
+        ))
     }
 
     pub async fn list_object_parts_query(
@@ -285,7 +348,19 @@ impl TransitionClient {
         part_number_marker: i64,
         max_parts: i64,
     ) -> Result<ListObjectPartsResult, std::io::Error> {
-        todo!();
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!(
+                        "list_object_parts_query is not implemented for bucket {bucket_name}, object {object_name}, upload_id {upload_id}"
+                    ),
+                },
+                request_id: "".to_string(),
+            },
+        ))
     }
 }
 
@@ -304,7 +379,44 @@ pub struct ListObjectsOptions {
 
 impl ListObjectsOptions {
     pub fn set(&mut self, key: &str, value: &str) {
-        todo!();
+        match key {
+            "prefix" => {
+                self.prefix = value.to_string();
+            }
+            "start-after" => {
+                self.start_after = value.to_string();
+            }
+            "max-keys" => {
+                if let Ok(v) = value.parse::<i64>() {
+                    self.max_keys = v;
+                }
+            }
+            "delimiter" => {
+                // delimiter is currently kept in request only; this option structure does not persist it yet.
+            }
+            "reverse" | "versions" | "metadata" | "recursive" | "use-v1" => {
+                if let Some(v) = value.strip_prefix("v").or_else(|| value.strip_prefix("V")) {
+                    let v = v.eq_ignore_ascii_case("true");
+                    match key {
+                        "reverse" => self.reverse_versions = v,
+                        "versions" => self.with_versions = v,
+                        "metadata" => self.with_metadata = v,
+                        "recursive" => self.recursive = v,
+                        _ => self.use_v1 = v,
+                    }
+                } else {
+                    let v = value.eq_ignore_ascii_case("true");
+                    match key {
+                        "reverse" => self.reverse_versions = v,
+                        "versions" => self.with_versions = v,
+                        "metadata" => self.with_metadata = v,
+                        "recursive" => self.recursive = v,
+                        _ => self.use_v1 = v,
+                    }
+                }
+            }
+            _ => {}
+        }
     }
 }
 

crates/ecstore/src/client/api_put_object_common.rs

@@ -27,11 +27,11 @@ use crate::client::{
 };
 
 pub fn is_object(reader: &ReaderImpl) -> bool {
-    todo!();
+    matches!(reader, ReaderImpl::ObjectBody(_))
 }
 
 pub fn is_read_at(reader: ReaderImpl) -> bool {
-    todo!();
+    matches!(reader, ReaderImpl::ObjectBody(_))
 }
 
 pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {

crates/ecstore/src/client/api_put_object_streaming.rs

@@ -97,11 +97,11 @@ impl TransitionClient {
         if opts.checksum.is_set() {
             opts.auto_checksum = opts.checksum.clone();
         }
-        let with_checksum = self.trailing_header_support;
-        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+
         opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
 
-        todo!();
+        self.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, &opts)
+            .await
     }
 
     pub async fn put_object_multipart_stream_optional_checksum(

crates/ecstore/src/client/api_remove.rs

@@ -26,14 +26,19 @@ use rustfs_utils::HashAlgorithm;
 use s3s::S3ErrorCode;
 use s3s::dto::ReplicationStatus;
 use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
+use serde::Deserialize;
 use std::fmt::Display;
-use std::{collections::HashMap, sync::Arc};
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+};
 use time::OffsetDateTime;
 use tokio::sync::mpsc::{self, Receiver, Sender};
 
 use crate::client::utils::base64_encode;
 use crate::client::{
     api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
+    api_s3_datatypes::{DeleteMultiObjects, DeleteObject},
     transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
 };
 use crate::{
@@ -46,6 +51,8 @@ pub struct RemoveBucketOptions {
     _forced_delete: bool,
 }
 
+const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
+
 #[derive(Debug)]
 #[allow(dead_code)]
 pub struct AdvancedRemoveOptions {
@@ -354,7 +361,7 @@ impl TransitionClient {
                     body_vec.extend_from_slice(data);
                 }
             }
-            process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_vec)), result_tx.clone());
+            process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_vec)), &batch, result_tx.clone()).await;
         }
         Ok(())
     }
@@ -498,11 +505,239 @@ pub struct RemoveObjectsOptions {
 }
 
 pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec<u8> {
-    todo!();
+    let escape_xml = |value: &str| -> String {
+        value
+            .replace('&', "&amp;")
+            .replace('<', "&lt;")
+            .replace('>', "&gt;")
+            .replace('\"', "&quot;")
+            .replace('\'', "&apos;")
+    };
+
+    let request: DeleteMultiObjects = DeleteMultiObjects {
+        quiet: false,
+        objects: objects
+            .iter()
+            .map(|object| DeleteObject {
+                key: object.name.clone(),
+                version_id: object.version_id.map(|v| v.to_string()).unwrap_or_default(),
+            })
+            .collect(),
+    };
+
+    match request.marshal_msg() {
+        Ok(body) => body.into_bytes(),
+        Err(_) => {
+            let mut body = String::new();
+            body.push_str("<Delete><Quiet>false</Quiet>");
+            for object in objects {
+                body.push_str("<Object>");
+                body.push_str("<Key>");
+                body.push_str(&escape_xml(&object.name));
+                body.push_str("</Key>");
+                if object.version_id.is_some() {
+                    body.push_str("<VersionId>");
+                    body.push_str(&escape_xml(&object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default()));
+                    body.push_str("</VersionId>");
+                }
+                body.push_str("</Object>");
+            }
+            body.push_str("</Delete>");
+            body.into_bytes()
+        }
+    }
 }
 
-pub fn process_remove_multi_objects_response(body: ReaderImpl, result_tx: Sender<RemoveObjectResult>) {
-    todo!();
+pub async fn process_remove_multi_objects_response(
+    body: ReaderImpl,
+    objects: &[ObjectInfo],
+    result_tx: Sender<RemoveObjectResult>,
+) {
+    let mut body_vec = Vec::new();
+    match body {
+        ReaderImpl::Body(content_body) => {
+            body_vec = content_body.to_vec();
+        }
+        ReaderImpl::ObjectBody(mut object_body) => match object_body.read_all().await {
+            Ok(content) => {
+                body_vec = content;
+            }
+            Err(err) => {
+                for object in objects {
+                    let version_id = object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default();
+                    let _ = result_tx
+                        .send(RemoveObjectResult {
+                            object_name: object.name.clone(),
+                            object_version_id: version_id,
+                            err: Some(std::io::Error::other(ErrorResponse {
+                                code: S3ErrorCode::Custom("ReadDeleteResponseFailed".into()),
+                                message: format!("read multi remove response failed: {err}"),
+                                bucket_name: object.bucket.clone(),
+                                key: object.name.clone(),
+                                resource: "".to_string(),
+                                request_id: "".to_string(),
+                                host_id: "".to_string(),
+                                region: "".to_string(),
+                                server: "".to_string(),
+                                status_code: StatusCode::OK,
+                            })),
+                            ..Default::default()
+                        })
+                        .await;
+                }
+                return;
+            }
+        },
+    }
+
+    #[derive(Debug, Deserialize)]
+    #[serde(rename = "DeleteResult")]
+    struct Deleted {
+        #[serde(rename = "Deleted", default)]
+        deleted: Vec<DeleteResultDeleted>,
+        #[serde(rename = "Error", default)]
+        error: Vec<DeleteResultError>,
+    }
+
+    #[derive(Debug, Deserialize)]
+    struct DeleteResultDeleted {
+        #[serde(rename = "Key")]
+        key: String,
+        #[serde(rename = "VersionId", default)]
+        version_id: String,
+        #[serde(rename = "DeleteMarker")]
+        deletemarker: bool,
+        #[serde(rename = "DeleteMarkerVersionId", default)]
+        deletemarker_version_id: String,
+    }
+
+    #[derive(Debug, Deserialize)]
+    struct DeleteResultError {
+        #[serde(rename = "Key")]
+        key: String,
+        #[serde(rename = "VersionId", default)]
+        version_id: String,
+        #[serde(rename = "Code")]
+        code: String,
+        #[serde(rename = "Message")]
+        message: String,
+    }
+
+    let mut pending = HashSet::with_capacity(objects.len());
+    for object in objects {
+        pending.insert((object.name.clone(), object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default()));
+    }
+
+    let body = String::from_utf8_lossy(&body_vec).into_owned();
+    let parsed: Deleted = match quick_xml::de::from_str(&body) {
+        Ok(parsed) => parsed,
+        Err(err) => {
+            for object in objects {
+                let version_id = object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default();
+                let _ = result_tx
+                    .send(RemoveObjectResult {
+                        object_name: object.name.clone(),
+                        object_version_id: version_id,
+                        err: Some(std::io::Error::other(ErrorResponse {
+                            code: S3ErrorCode::Custom("UnmarshalDeleteResponseFailed".into()),
+                            message: format!(
+                                "unmarshal multi remove response failed: {err}; response_body={}",
+                                body.chars().take(DELETE_RESPONSE_PREVIEW_LEN).collect::<String>()
+                            ),
+                            bucket_name: object.bucket.clone(),
+                            key: object.name.clone(),
+                            resource: "".to_string(),
+                            request_id: "".to_string(),
+                            host_id: "".to_string(),
+                            region: "".to_string(),
+                            server: "".to_string(),
+                            status_code: StatusCode::OK,
+                        })),
+                        ..Default::default()
+                    })
+                    .await;
+            }
+            return;
+        }
+    };
+
+    for deleted in parsed.deleted {
+        if !pending.remove(&(deleted.key.clone(), deleted.version_id.clone())) {
+            continue;
+        }
+
+        let _ = result_tx
+            .send(RemoveObjectResult {
+                object_name: deleted.key,
+                object_version_id: deleted.version_id,
+                delete_marker: deleted.deletemarker,
+                delete_marker_version_id: deleted.deletemarker_version_id,
+                err: None,
+            })
+            .await;
+    }
+
+    for removed in parsed.error {
+        if !pending.remove(&(removed.key.clone(), removed.version_id.clone())) {
+            continue;
+        }
+
+        let _ = result_tx
+            .send(RemoveObjectResult {
+                object_name: removed.key.clone(),
+                object_version_id: removed.version_id,
+                err: Some(std::io::Error::other(ErrorResponse {
+                    code: S3ErrorCode::Custom(removed.code.into()),
+                    message: removed.message,
+                    bucket_name: "".to_string(),
+                    key: removed.key,
+                    resource: "".to_string(),
+                    request_id: "".to_string(),
+                    host_id: "".to_string(),
+                    region: "".to_string(),
+                    server: "".to_string(),
+                    status_code: StatusCode::OK,
+                })),
+                ..Default::default()
+            })
+            .await;
+    }
+
+    for (object_name, object_version_id) in pending {
+        let bucket_name = objects
+            .iter()
+            .find(|object| {
+                object.name == object_name && object.version_id.as_ref().map(|v| v.to_string()) == Some(object_version_id.clone())
+            })
+            .map(|o| o.bucket.clone())
+            .unwrap_or_default();
+        let object_name = object_name;
+        let object_version_id = object_version_id;
+        let error_message = format!(
+            "remove response did not contain an entry for object {} with version {}",
+            object_name, object_version_id
+        );
+
+        let _ = result_tx
+            .send(RemoveObjectResult {
+                object_name: object_name.clone(),
+                object_version_id: object_version_id.clone(),
+                err: Some(std::io::Error::other(ErrorResponse {
+                    code: S3ErrorCode::Custom("UnmatchedDeleteResponseEntry".into()),
+                    message: error_message,
+                    bucket_name,
+                    key: object_name,
+                    resource: "".to_string(),
+                    request_id: "".to_string(),
+                    host_id: "".to_string(),
+                    region: "".to_string(),
+                    server: "".to_string(),
+                    status_code: StatusCode::OK,
+                })),
+                ..Default::default()
+            })
+            .await;
+    }
 }
 
 fn has_invalid_xml_char(str: &str) -> bool {

crates/ecstore/src/client/api_s3_datatypes.rs

@@ -290,7 +290,50 @@ impl CompleteMultipartUpload {
     }
 
     pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
-        todo!();
+        #[derive(Debug, Deserialize)]
+        struct WirePart {
+            #[serde(rename = "ETag")]
+            etag: String,
+            #[serde(rename = "PartNumber")]
+            part_num: i64,
+            #[serde(rename = "ChecksumCRC32")]
+            checksum_crc32: String,
+            #[serde(rename = "ChecksumCRC32C")]
+            checksum_crc32c: String,
+            #[serde(rename = "ChecksumSHA1")]
+            checksum_sha1: String,
+            #[serde(rename = "ChecksumSHA256")]
+            checksum_sha256: String,
+            #[serde(rename = "ChecksumCRC64NVME")]
+            checksum_crc64nvme: String,
+        }
+
+        #[derive(Debug, Deserialize)]
+        #[serde(rename = "CompleteMultipartUpload")]
+        struct WireCompleteMultipartUpload {
+            #[serde(rename = "Part", default)]
+            parts: Vec<WirePart>,
+        }
+
+        let body = String::from_utf8_lossy(buf);
+        let wire: WireCompleteMultipartUpload = quick_xml::de::from_str(&body)
+            .map_err(|err| std::io::Error::other(format!("failed to parse CompleteMultipartUpload XML: {err}; body: {body}")))?;
+
+        Ok(Self {
+            parts: wire
+                .parts
+                .into_iter()
+                .map(|p| CompletePart {
+                    etag: p.etag,
+                    part_num: p.part_num,
+                    checksum_crc32: p.checksum_crc32,
+                    checksum_crc32c: p.checksum_crc32c,
+                    checksum_sha1: p.checksum_sha1,
+                    checksum_sha256: p.checksum_sha256,
+                    checksum_crc64nvme: p.checksum_crc64nvme,
+                })
+                .collect(),
+        })
     }
 }
 
@@ -340,7 +383,37 @@ impl DeleteMultiObjects {
     }
 
     pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
-        todo!();
+        #[derive(Debug, Deserialize)]
+        struct WireDeleteObject {
+            #[serde(rename = "Key")]
+            key: String,
+            #[serde(rename = "VersionId")]
+            version_id: String,
+        }
+
+        #[derive(Debug, Deserialize)]
+        #[serde(rename = "Delete")]
+        struct WireDeleteMultiObjects {
+            #[serde(rename = "Quiet", default)]
+            quiet: bool,
+            #[serde(rename = "Object", default)]
+            objects: Vec<WireDeleteObject>,
+        }
+
+        let body = String::from_utf8_lossy(buf);
+        let wire: WireDeleteMultiObjects = quick_xml::de::from_str(&body).map_err(|err| std::io::Error::other(err))?;
+
+        Ok(Self {
+            quiet: wire.quiet,
+            objects: wire
+                .objects
+                .into_iter()
+                .map(|o| DeleteObject {
+                    key: o.key,
+                    version_id: o.version_id,
+                })
+                .collect(),
+        })
     }
 }
 

crates/ecstore/src/client/checksum.rs

@@ -248,8 +248,12 @@ impl ChecksumMode {
             }
         });
         let c = self.base();
-        let crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
+        let mut crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
         let mut h = self.hasher()?;
+        for part in p.iter() {
+            let part_checksum = part.checksum_raw(&c)?;
+            crc_bytes.extend(part_checksum);
+        }
         h.update(crc_bytes.as_ref());
         let hash = h.finalize();
         Ok(Checksum {
@@ -260,7 +264,11 @@ impl ChecksumMode {
     }
 
     pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
-        todo!();
+        if !self.can_merge_crc() {
+            return Err(std::io::Error::other("cannot do full-object checksum"));
+        }
+
+        self.composite_checksum(p)
     }
 }
 

crates/ecstore/src/client/credentials.rs

@@ -18,7 +18,10 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]
 
+use quick_xml;
+use serde::de::Deserialize;
 use std::fmt::{Display, Formatter};
+use std::io::{Error, ErrorKind};
 
 use time::OffsetDateTime;
 
@@ -154,10 +157,21 @@ impl ErrorResponse {
     }
 }
 
-pub fn xml_decoder<T>(body: &[u8]) -> Result<T, std::io::Error> {
-    todo!();
+pub fn xml_decoder<T>(body: &[u8]) -> Result<T, Error>
+where
+    for<'de> T: Deserialize<'de>,
+{
+    match std::str::from_utf8(body) {
+        Ok(xml_body) => quick_xml::de::from_str::<T>(xml_body).map_err(|err| Error::new(ErrorKind::InvalidData, err.to_string())),
+        Err(err) => Err(Error::new(ErrorKind::InvalidData, err.to_string())),
+    }
 }
 
-pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error> {
-    todo!();
+pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error>
+where
+    for<'de> T: Deserialize<'de>,
+{
+    let body = body_reader.to_vec();
+    let parsed = xml_decoder(&body)?;
+    Ok((body, parsed))
 }

crates/ecstore/src/client/object_api_utils.rs

@@ -33,18 +33,17 @@ use s3s::S3ErrorCode;
 //#[derive(Clone)]
 pub struct PutObjReader {
     pub reader: HashReader,
-    pub raw_reader: HashReader,
     //pub sealMD5Fn: SealMD5CurrFn,
 }
 
 #[allow(dead_code)]
 impl PutObjReader {
-    pub fn new(raw_reader: HashReader) -> Self {
-        todo!();
+    pub fn new(reader: HashReader) -> Self {
+        Self { reader }
     }
 
     fn md5_current_hex_string(&self) -> String {
-        todo!();
+        self.reader.checksum().map(|v| v.encoded).unwrap_or_default()
     }
 
     fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
@@ -100,9 +99,11 @@ fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u
     let parts: &[ObjectPartInfo] = &oi.parts;
     if skip_length > 0
         && parts.len() > first_part_idx as usize
-        && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0
+        && parts[first_part_idx as usize].index.as_ref().is_some_and(|idx| idx.len() > 0)
     {
-        todo!();
+        let _ = part_skip;
+        let _ = decrypt_skip;
+        let _ = seq_num;
     }
 
     (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)

crates/ecstore/src/client/object_handlers_common.rs

@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::StorageAPI;
 use crate::bucket::lifecycle::lifecycle;
 use crate::bucket::versioning::VersioningApi;
 use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::store::ECStore;
-use crate::store_api::{ObjectOptions, ObjectToDelete};
+use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete};
 use rustfs_lock::MAX_DELETE_LIST;
 
 pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {

crates/ecstore/src/client/transition_api.rs

@@ -165,7 +165,6 @@ impl TransitionClient {
     async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
         let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
 
-        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
         let scheme = endpoint_url.scheme();
         let client;
         let tls = if let Some(store) = load_root_store_from_tls_path() {
@@ -243,11 +242,13 @@ impl TransitionClient {
     }
 
     fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
-        todo!();
+        let mut endpoint = self.s3_accelerate_endpoint.lock().unwrap();
+        *endpoint = accelerate_endpoint.to_string();
     }
 
     fn set_s3_enable_dual_stack(&self, enabled: bool) {
-        todo!();
+        let mut dual_stack = self.s3_dual_stack_enabled.lock().unwrap();
+        *dual_stack = enabled;
     }
 
     pub fn hash_materials(
@@ -255,7 +256,22 @@ impl TransitionClient {
         is_md5_requested: bool,
         is_sha256_requested: bool,
     ) -> (HashMap<String, HashAlgorithm>, HashMap<String, Vec<u8>>) {
-        todo!()
+        // `hash_algos` declares which algorithms are active for this multipart upload.
+        // `hash_sums` keeps the current part digest bytes and is refreshed on every loop.
+        let mut hash_algos = HashMap::new();
+        let mut hash_sums = HashMap::new();
+
+        if is_md5_requested {
+            hash_algos.insert("md5".to_string(), HashAlgorithm::Md5);
+            hash_sums.insert("md5".to_string(), vec![]);
+        }
+
+        if is_sha256_requested {
+            hash_algos.insert("sha256".to_string(), HashAlgorithm::SHA256);
+            hash_sums.insert("sha256".to_string(), vec![]);
+        }
+
+        (hash_algos, hash_sums)
     }
 
     fn is_online(&self) -> bool {
@@ -272,10 +288,10 @@ impl TransitionClient {
     }
 
     fn health_check(hc_duration: Duration) {
-        todo!();
+        let _ = hc_duration;
     }
 
-    fn dump_http(&self, req: &http::Request<s3s::Body>, resp: &http::Response<Incoming>) -> Result<(), std::io::Error> {
+    fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> {
         let mut resp_trace: Vec<u8>;
 
         //info!("{}{}", self.trace_output, "---------BEGIN-HTTP---------");
@@ -284,7 +300,7 @@ impl TransitionClient {
         Ok(())
     }
 
-    pub async fn doit(&self, req: http::Request<s3s::Body>) -> Result<http::Response<Incoming>, std::io::Error> {
+    pub async fn doit(&self, req: Request<s3s::Body>) -> Result<Response<Incoming>, std::io::Error> {
         let req_method;
         let req_uri;
         let req_headers;
@@ -336,7 +352,7 @@ impl TransitionClient {
         &self,
         method: http::Method,
         metadata: &mut RequestMetadata,
-    ) -> Result<http::Response<Incoming>, std::io::Error> {
+    ) -> Result<Response<Incoming>, std::io::Error> {
         if self.is_offline() {
             let mut s = self.endpoint_url.to_string();
             s.push_str(" is offline.");
@@ -346,7 +362,7 @@ impl TransitionClient {
         let retryable: bool;
         //let mut body_seeker: BufferReader;
         let mut req_retry = self.max_retries;
-        let mut resp: http::Response<Incoming>;
+        let mut resp: Response<Incoming>;
 
         //if metadata.content_body != nil {
         //body_seeker = BufferReader::new(metadata.content_body.read_all().await?);
@@ -384,10 +400,10 @@ impl TransitionClient {
             err_response.message = format!("remote tier error: {}", err_response.message);
 
             if self.region == "" {
-                match err_response.code {
+                return match err_response.code {
                     S3ErrorCode::AuthorizationHeaderMalformed | S3ErrorCode::InvalidArgument /*S3ErrorCode::InvalidRegion*/ => {
                         //break;
-                        return Err(std::io::Error::other(err_response));
+                        Err(std::io::Error::other(err_response))
                     }
                     S3ErrorCode::AccessDenied => {
                         if err_response.region == "" {
@@ -404,12 +420,12 @@ impl TransitionClient {
                             metadata.bucket_location = err_response.region.clone();
                             //continue;
                         }
-                        return Err(std::io::Error::other(err_response));
+                        Err(std::io::Error::other(err_response))
                     }
                     _ => {
-                        return Err(std::io::Error::other(err_response));
+                        Err(std::io::Error::other(err_response))
                     }
-                }
+                };
             }
 
             if is_s3code_retryable(err_response.code.as_str()) {
@@ -430,7 +446,7 @@ impl TransitionClient {
         &self,
         method: &http::Method,
         metadata: &mut RequestMetadata,
-    ) -> Result<http::Request<s3s::Body>, std::io::Error> {
+    ) -> Result<Request<s3s::Body>, std::io::Error> {
         let mut location = metadata.bucket_location.clone();
         if location == "" && metadata.bucket_name != "" {
             location = self.get_bucket_location(&metadata.bucket_name).await?;
@@ -734,7 +750,19 @@ impl TransitionCore {
     ) -> Result<CompletePart, std::io::Error> {
         //self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id,
         //    part_id, start_offset, length, metadata)
-        todo!();
+        Err(std::io::Error::new(
+            std::io::ErrorKind::Unsupported,
+            crate::client::credentials::ErrorResponse {
+                sts_error: crate::client::credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!(
+                        "copy_object_part is not implemented for {src_bucket}/{src_object} -> {dest_bucket}/{dest_object}"
+                    ),
+                },
+                request_id: "".to_string(),
+            },
+        ))
     }
 
     pub async fn put_object(

crates/ecstore/src/compress.rs

@@ -14,38 +14,93 @@
 
 use rustfs_utils::string::{has_pattern, has_string_suffix_in_slice};
 use std::env;
-use tracing::error;
+use std::sync::OnceLock;
+use tracing::debug;
 
 pub const MIN_COMPRESSIBLE_SIZE: usize = 4096;
 
 // Environment variable name to control whether compression is enabled
 pub const ENV_COMPRESSION_ENABLED: &str = "RUSTFS_COMPRESSION_ENABLED";
 
+// Environment variable for additional extensions to exclude from compression (comma-separated, e.g. ".foo,.bar")
+pub const ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS: &str = "RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS";
+
+/// Parses RUSTFS_COMPRESSION_ENABLED. Called once at first use via OnceLock.
+fn parse_compression_enabled() -> bool {
+    env::var(ENV_COMPRESSION_ENABLED)
+        .map(|s| s.to_lowercase() == "true")
+        .unwrap_or(false)
+}
+
+/// Parses RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS (comma-separated). Called once at first use via OnceLock.
+pub(crate) fn parse_added_exclude_extensions() -> Vec<String> {
+    env::var(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS)
+        .ok()
+        .map(|s| {
+            s.split(',')
+                .map(|v| {
+                    let v = v.trim().to_lowercase();
+                    if v.is_empty() {
+                        String::new()
+                    } else if v.starts_with('.') {
+                        v
+                    } else {
+                        format!(".{v}")
+                    }
+                })
+                .filter(|v| !v.is_empty())
+                .collect()
+        })
+        .unwrap_or_default()
+}
+
+// Parsed once at first use, then reused for all is_compressible() checks.
+static COMPRESSION_ENABLED: OnceLock<bool> = OnceLock::new();
+static ADDED_EXCLUDE_COMPRESS_EXTENSIONS: OnceLock<Vec<String>> = OnceLock::new();
+
 // Some standard object extensions which we strictly dis-allow for compression.
+#[rustfmt::skip]
 pub const STANDARD_EXCLUDE_COMPRESS_EXTENSIONS: &[&str] = &[
-    ".gz", ".bz2", ".rar", ".zip", ".7z", ".xz", ".mp4", ".mkv", ".mov", ".jpg", ".png", ".gif",
+    // Compressed archives
+    ".gz", ".bz2", ".rar", ".zip", ".7z", ".xz", ".zst", ".lz4", ".br", ".lzo", ".sz", ".tgz",
+    // Images
+    ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".heic", ".heif", ".jxl",
+    // Video
+    ".mp4", ".mkv", ".mov", ".avi", ".wmv", ".flv", ".webm", ".m4v", ".mpeg", ".mpg",
+    // Audio
+    ".mp3", ".aac", ".ogg", ".flac", ".wma", ".m4a", ".opus",
+    // Documents (internally compressed)
+    ".pdf", ".docx", ".xlsx", ".pptx",
+    // Package formats
+    ".deb", ".rpm", ".jar", ".war", ".apk",
+    // Web fonts
+    ".woff", ".woff2",
 ];
 
 // Some standard content-types which we strictly dis-allow for compression.
 pub const STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES: &[&str] = &[
     "video/*",
     "audio/*",
+    "image/*",
     "application/zip",
     "application/x-gzip",
     "application/x-zip-compressed",
     "application/x-compress",
     "application/x-spoon",
+    "application/x-rar-compressed",
+    "application/x-7z-compressed",
+    "application/x-bzip2",
+    "application/x-xz",
+    "application/zstd",
+    "application/pdf",
+    "application/wasm",
+    "font/*",
 ];
 
 pub fn is_compressible(headers: &http::HeaderMap, object_name: &str) -> bool {
-    // Check if compression is enabled via environment variable, default disabled
-    if let Ok(compression_enabled) = env::var(ENV_COMPRESSION_ENABLED) {
-        if compression_enabled.to_lowercase() != "true" {
-            error!("Compression is disabled by environment variable");
-            return false;
-        }
-    } else {
-        // Default disabled when environment variable is not set
+    // Check if compression is enabled (read once at first use, then fixed for process lifetime)
+    if !*COMPRESSION_ENABLED.get_or_init(parse_compression_enabled) {
+        debug!("Compression is disabled by environment variable");
         return false;
     }
 
@@ -54,12 +109,18 @@ pub fn is_compressible(headers: &http::HeaderMap, object_name: &str) -> bool {
     // TODO: crypto request return false
 
     if has_string_suffix_in_slice(object_name, STANDARD_EXCLUDE_COMPRESS_EXTENSIONS) {
-        error!("object_name: {} is not compressible", object_name);
+        debug!("object_name: {} is not compressible", object_name);
+        return false;
+    }
+
+    let added = ADDED_EXCLUDE_COMPRESS_EXTENSIONS.get_or_init(parse_added_exclude_extensions);
+    if !added.is_empty() && has_string_suffix_in_slice(object_name, added) {
+        debug!("object_name: {} is not compressible (added exclusion)", object_name);
         return false;
     }
 
     if !content_type.is_empty() && has_pattern(STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES, content_type) {
-        error!("content_type: {} is not compressible", content_type);
+        debug!("content_type: {} is not compressible", content_type);
         return false;
     }
     true
@@ -72,33 +133,78 @@ mod tests {
     use super::*;
     use temp_env;
 
+    #[test]
+    fn test_parse_compression_enabled() {
+        temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || {
+            assert!(parse_compression_enabled());
+        });
+        temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("false"), || {
+            assert!(!parse_compression_enabled());
+        });
+        temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("FALSE"), || {
+            assert!(!parse_compression_enabled());
+        });
+        temp_env::with_var_unset(ENV_COMPRESSION_ENABLED, || {
+            assert!(!parse_compression_enabled());
+        });
+    }
+
     #[test]
     fn test_is_compressible() {
         use http::HeaderMap;
 
         let headers = HeaderMap::new();
 
-        // Test environment variable control
-        temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("false"), || {
-            assert!(!is_compressible(&headers, "file.txt"));
-        });
-
+        // COMPRESSION_ENABLED is cached in OnceLock at first use; test with "true" so extension/content-type logic is exercised.
+        // For env false/unset behavior see test_parse_compression_enabled.
         temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || {
             assert!(is_compressible(&headers, "file.txt"));
         });
 
-        temp_env::with_var_unset(ENV_COMPRESSION_ENABLED, || {
-            assert!(!is_compressible(&headers, "file.txt"));
-        });
-
         temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || {
             let mut headers = HeaderMap::new();
-            // Test non-compressible extensions
+            // Test non-compressible extensions - compressed archives
             headers.insert("content-type", "text/plain".parse().unwrap());
             assert!(!is_compressible(&headers, "file.gz"));
             assert!(!is_compressible(&headers, "file.zip"));
-            assert!(!is_compressible(&headers, "file.mp4"));
+            assert!(!is_compressible(&headers, "file.7z"));
+            assert!(!is_compressible(&headers, "file.zst"));
+            assert!(!is_compressible(&headers, "file.br"));
+            assert!(!is_compressible(&headers, "file.tgz"));
+
+            // Test non-compressible extensions - images
             assert!(!is_compressible(&headers, "file.jpg"));
+            assert!(!is_compressible(&headers, "file.jpeg"));
+            assert!(!is_compressible(&headers, "file.png"));
+            assert!(!is_compressible(&headers, "file.gif"));
+            assert!(!is_compressible(&headers, "file.webp"));
+            assert!(!is_compressible(&headers, "file.avif"));
+
+            // Test non-compressible extensions - video
+            assert!(!is_compressible(&headers, "file.mp4"));
+            assert!(!is_compressible(&headers, "file.mkv"));
+            assert!(!is_compressible(&headers, "file.webm"));
+
+            // Test non-compressible extensions - audio
+            assert!(!is_compressible(&headers, "file.mp3"));
+            assert!(!is_compressible(&headers, "file.aac"));
+            assert!(!is_compressible(&headers, "file.ogg"));
+            assert!(!is_compressible(&headers, "file.flac"));
+            assert!(!is_compressible(&headers, "file.opus"));
+
+            // Test non-compressible extensions - documents
+            assert!(!is_compressible(&headers, "file.pdf"));
+            assert!(!is_compressible(&headers, "file.docx"));
+            assert!(!is_compressible(&headers, "file.xlsx"));
+            assert!(!is_compressible(&headers, "file.pptx"));
+
+            // Test non-compressible extensions - packages
+            assert!(!is_compressible(&headers, "file.deb"));
+            assert!(!is_compressible(&headers, "file.rpm"));
+            assert!(!is_compressible(&headers, "file.jar"));
+
+            // Test non-compressible extensions - web fonts
+            assert!(!is_compressible(&headers, "file.woff2"));
 
             // Test non-compressible content types
             headers.insert("content-type", "video/mp4".parse().unwrap());
@@ -107,12 +213,27 @@ mod tests {
             headers.insert("content-type", "audio/mpeg".parse().unwrap());
             assert!(!is_compressible(&headers, "file.txt"));
 
+            headers.insert("content-type", "image/png".parse().unwrap());
+            assert!(!is_compressible(&headers, "file.txt"));
+
+            headers.insert("content-type", "image/webp".parse().unwrap());
+            assert!(!is_compressible(&headers, "file.txt"));
+
             headers.insert("content-type", "application/zip".parse().unwrap());
             assert!(!is_compressible(&headers, "file.txt"));
 
             headers.insert("content-type", "application/x-gzip".parse().unwrap());
             assert!(!is_compressible(&headers, "file.txt"));
 
+            headers.insert("content-type", "application/x-7z-compressed".parse().unwrap());
+            assert!(!is_compressible(&headers, "file.txt"));
+
+            headers.insert("content-type", "application/pdf".parse().unwrap());
+            assert!(!is_compressible(&headers, "file.txt"));
+
+            headers.insert("content-type", "font/woff2".parse().unwrap());
+            assert!(!is_compressible(&headers, "file.txt"));
+
             // Test compressible cases
             headers.insert("content-type", "text/plain".parse().unwrap());
             assert!(is_compressible(&headers, "file.txt"));
@@ -125,4 +246,34 @@ mod tests {
             assert!(is_compressible(&headers, "file.json"));
         });
     }
+
+    #[test]
+    fn test_added_exclude_compress_extensions_parsing() {
+        temp_env::with_var(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, Some(".foo,.bar"), || {
+            let added = parse_added_exclude_extensions();
+            assert_eq!(added, [".foo", ".bar"]);
+        });
+        temp_env::with_var(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, Some("baz, .qux"), || {
+            let added = parse_added_exclude_extensions();
+            assert_eq!(added, [".baz", ".qux"]);
+        });
+        temp_env::with_var_unset(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, || {
+            let added = parse_added_exclude_extensions();
+            assert!(added.is_empty());
+        });
+    }
+
+    #[test]
+    fn test_added_exclude_compress_extensions_default_compressible() {
+        use http::HeaderMap;
+
+        let mut headers = HeaderMap::new();
+        headers.insert("content-type", "text/plain".parse().unwrap());
+        temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || {
+            temp_env::with_var_unset(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, || {
+                assert!(is_compressible(&headers, "file.foo"));
+                assert!(is_compressible(&headers, "file.bar"));
+            });
+        });
+    }
 }

crates/ecstore/src/config/mod.rs

@@ -17,6 +17,7 @@ pub mod com;
 #[allow(dead_code)]
 pub mod heal;
 mod notify;
+mod oidc;
 pub mod storageclass;
 
 use crate::error::Result;
@@ -26,6 +27,7 @@ use rustfs_config::COMMENT_KEY;
 use rustfs_config::DEFAULT_DELIMITER;
 use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
 use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::LazyLock;
@@ -65,6 +67,14 @@ impl ConfigSys {
     }
 }
 
+pub fn get_global_server_config() -> Option<Config> {
+    GLOBAL_SERVER_CONFIG.get().cloned()
+}
+
+pub async fn init_global_config_sys(api: Arc<ECStore>) -> Result<()> {
+    GLOBAL_CONFIG_SYS.init(api).await
+}
+
 #[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct KV {
     pub key: String,
@@ -226,6 +236,7 @@ pub fn init() {
     kvs.insert(AUDIT_WEBHOOK_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone());
     kvs.insert(NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_MQTT_KVS.clone());
     kvs.insert(AUDIT_MQTT_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MQTT_KVS.clone());
+    kvs.insert(IDENTITY_OPENID_SUB_SYS.to_owned(), oidc::DEFAULT_IDENTITY_OPENID_KVS.clone());
 
     // Register all default configurations
     register_default_kvs(kvs)

crates/ecstore/src/config/oidc.rs

@@ -0,0 +1,101 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::config::{KV, KVS};
+use rustfs_config::{
+    ENABLE_KEY, EnableState,
+    oidc::{
+        OIDC_CLAIM_NAME, OIDC_CLAIM_PREFIX, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_CONFIG_URL, OIDC_DEFAULT_CLAIM_NAME,
+        OIDC_DEFAULT_EMAIL_CLAIM, OIDC_DEFAULT_GROUPS_CLAIM, OIDC_DEFAULT_SCOPES, OIDC_DEFAULT_USERNAME_CLAIM, OIDC_DISPLAY_NAME,
+        OIDC_EMAIL_CLAIM, OIDC_GROUPS_CLAIM, OIDC_REDIRECT_URI, OIDC_REDIRECT_URI_DYNAMIC, OIDC_ROLE_POLICY, OIDC_SCOPES,
+        OIDC_USERNAME_CLAIM,
+    },
+};
+use std::sync::LazyLock;
+
+/// Default KVS for OIDC identity provider settings.
+pub static DEFAULT_IDENTITY_OPENID_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_CONFIG_URL.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_CLIENT_ID.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_CLIENT_SECRET.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: OIDC_SCOPES.to_owned(),
+            value: OIDC_DEFAULT_SCOPES.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_REDIRECT_URI.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_REDIRECT_URI_DYNAMIC.to_owned(),
+            value: EnableState::On.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_CLAIM_NAME.to_owned(),
+            value: OIDC_DEFAULT_CLAIM_NAME.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_CLAIM_PREFIX.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_ROLE_POLICY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_DISPLAY_NAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_GROUPS_CLAIM.to_owned(),
+            value: OIDC_DEFAULT_GROUPS_CLAIM.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_EMAIL_CLAIM.to_owned(),
+            value: OIDC_DEFAULT_EMAIL_CLAIM.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OIDC_USERNAME_CLAIM.to_owned(),
+            value: OIDC_DEFAULT_USERNAME_CLAIM.to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/data_usage.rs

@@ -15,8 +15,12 @@
 pub mod local_snapshot;
 
 use crate::{
-    bucket::metadata_sys::get_replication_config, config::com::read_config, disk::DiskAPI, error::Error, store::ECStore,
-    store_api::StorageAPI,
+    bucket::metadata_sys::get_replication_config,
+    config::com::read_config,
+    disk::DiskAPI,
+    error::Error,
+    store::ECStore,
+    store_api::{BucketOperations, ListOperations},
 };
 pub use local_snapshot::{
     DATA_USAGE_DIR, DATA_USAGE_STATE_DIR, LOCAL_USAGE_SNAPSHOT_VERSION, LocalUsageSnapshot, LocalUsageSnapshotMeta,

crates/ecstore/src/disk/disk_store.rs

@@ -20,7 +20,6 @@ use crate::disk::{
 };
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
-use rustfs_utils::string::parse_bool_with_default;
 use std::{
     path::PathBuf,
     sync::{
@@ -39,6 +38,7 @@ const DISK_HEALTH_OK: u32 = 0;
 const DISK_HEALTH_FAULTY: u32 = 1;
 
 pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
+pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
 pub const ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
 pub const CHECK_EVERY: Duration = Duration::from_secs(15);
 pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
@@ -180,40 +180,37 @@ pub struct LocalDiskWrapper {
 impl LocalDiskWrapper {
     /// Create a new LocalDiskWrapper
     pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
-        // Check environment variable for health check override
-        // Default to true if not set, but only enable if both param and env are true
-        let env_health_check = std::env::var(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING)
-            .map(|v| parse_bool_with_default(&v, true))
-            .unwrap_or(true);
+        // Check environment variable for health check override.
+        // Only enable if both param and env are true.
+        let env_health_check =
+            rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
 
-        let ret = Self {
+        Self {
             disk,
             health: Arc::new(DiskHealthTracker::new()),
             health_check: health_check && env_health_check,
             cancel_token: CancellationToken::new(),
             disk_id: Arc::new(RwLock::new(None)),
-        };
-
-        ret.start_monitoring();
-
-        ret
+        }
     }
 
     pub fn get_disk(&self) -> Arc<LocalDisk> {
         self.disk.clone()
     }
 
-    /// Start the disk monitoring if health_check is enabled
-    pub fn start_monitoring(&self) {
-        if self.health_check {
-            let health = Arc::clone(&self.health);
-            let cancel_token = self.cancel_token.clone();
-            let disk = Arc::clone(&self.disk);
-
-            tokio::spawn(async move {
-                Self::monitor_disk_writable(disk, health, cancel_token).await;
-            });
+    /// Enable health monitoring after disk creation.
+    /// Used to defer health checks until after startup format loading completes.
+    pub fn enable_health_check(&self) {
+        if !self.health_check {
+            return;
         }
+        let health = Arc::clone(&self.health);
+        let cancel_token = self.cancel_token.clone();
+        let disk = Arc::clone(&self.disk);
+
+        tokio::spawn(async move {
+            Self::monitor_disk_writable(disk, health, cancel_token).await;
+        });
     }
 
     /// Stop the disk monitoring

crates/ecstore/src/disk/local.rs

@@ -339,9 +339,7 @@ impl LocalDisk {
 
     #[tracing::instrument(level = "debug", skip(self))]
     async fn check_format_json(&self) -> Result<Metadata> {
-        let md = tokio::fs::metadata(&self.format_path)
-            .await
-            .map_err(to_unformatted_disk_error)?;
+        let md = std::fs::metadata(&self.format_path).map_err(to_unformatted_disk_error)?;
         Ok(md)
     }
     async fn make_meta_volumes(&self) -> Result<()> {
@@ -976,6 +974,7 @@ impl LocalDisk {
         opts: &WalkDirOptions,
         out: &mut MetacacheWriter<W>,
         objs_returned: &mut i32,
+        skip_current_dir_object: bool,
     ) -> Result<()>
     where
         W: AsyncWrite + Unpin + Send,
@@ -1072,6 +1071,10 @@ impl LocalDisk {
             *item = "".to_owned();
 
             if entry.ends_with(STORAGE_FORMAT_FILE) {
+                if skip_current_dir_object {
+                    continue;
+                }
+
                 let metadata = self
                     .read_metadata(bucket, format!("{}/{}", &current, &entry).as_str())
                     .await?;
@@ -1094,7 +1097,7 @@ impl LocalDisk {
                 })
                 .await?;
 
-                return Ok(());
+                continue;
             }
         }
 
@@ -1109,7 +1112,7 @@ impl LocalDisk {
             }
         }
 
-        let mut dir_stack: Vec<String> = Vec::with_capacity(5);
+        let mut dir_stack: Vec<(String, bool)> = Vec::with_capacity(5);
         prefix = "".to_owned();
 
         for entry in entries.iter() {
@@ -1123,7 +1126,7 @@ impl LocalDisk {
 
             let name = path_join_buf(&[current.as_str(), entry.as_str()]);
 
-            while let Some(pop) = dir_stack.last().cloned()
+            while let Some((pop, skip_object)) = dir_stack.last().cloned()
                 && pop < name
             {
                 out.write_obj(&MetaCacheEntry {
@@ -1133,7 +1136,7 @@ impl LocalDisk {
                 .await?;
 
                 if opts.recursive
-                    && let Err(er) = Box::pin(self.scan_dir(pop, prefix.clone(), opts, out, objs_returned)).await
+                    && let Err(er) = Box::pin(self.scan_dir(pop, prefix.clone(), opts, out, objs_returned, skip_object)).await
                 {
                     error!("scan_dir err {:?}", er);
                 }
@@ -1172,6 +1175,14 @@ impl LocalDisk {
                     // {
                     *objs_returned += 1;
                     // }
+
+                    if opts.recursive {
+                        let mut dir_name = meta.name.clone();
+                        if !dir_name.ends_with(SLASH_SEPARATOR) {
+                            dir_name.push_str(SLASH_SEPARATOR);
+                        }
+                        dir_stack.push((dir_name, true));
+                    }
                 }
                 Err(err) => {
                     if err == Error::FileNotFound || err == Error::IsNotRegular {
@@ -1179,7 +1190,7 @@ impl LocalDisk {
                         // If dirObject, but no metadata (which is unexpected) we skip it.
                         if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await {
                             meta.name.push_str(SLASH_SEPARATOR);
-                            dir_stack.push(meta.name);
+                            dir_stack.push((meta.name, false));
                         }
                     }
 
@@ -1188,7 +1199,7 @@ impl LocalDisk {
             };
         }
 
-        while let Some(dir) = dir_stack.pop() {
+        while let Some((dir, skip_object)) = dir_stack.pop() {
             if opts.limit > 0 && *objs_returned >= opts.limit {
                 return Ok(());
             }
@@ -1200,7 +1211,7 @@ impl LocalDisk {
             .await?;
 
             if opts.recursive
-                && let Err(er) = Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned)).await
+                && let Err(er) = Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned, skip_object)).await
             {
                 warn!("scan_dir err {:?}", &er);
             }
@@ -1367,43 +1378,36 @@ impl DiskAPI for LocalDisk {
 
     #[tracing::instrument(level = "debug", skip(self))]
     async fn get_disk_id(&self) -> Result<Option<Uuid>> {
-        let (id, last_check, file_info) = {
+        let format_info = {
             let format_info = self.format_info.read().await;
-            (format_info.id, format_info.last_check, format_info.file_info.clone())
+            format_info.clone()
         };
 
-        // Check if we can use cached value without doing any I/O
-        // If we checked recently (within 1 second) and have valid cache, return immediately
-        if let (Some(id), Some(last_check)) = (id, last_check)
-            && last_check.unix_timestamp() + 1 >= OffsetDateTime::now_utc().unix_timestamp()
-        {
-            return Ok(Some(id));
+        let id = format_info.id;
+
+        // if format_info.last_check_valid() {
+        //     return Ok(id);
+        // }
+
+        if format_info.file_info.is_some() && id.is_some() {
+            // check last check time
+            if let Some(last_check) = format_info.last_check
+                && last_check.unix_timestamp() + 1 < OffsetDateTime::now_utc().unix_timestamp()
+            {
+                return Ok(id);
+            }
         }
 
-        // Get current file metadata (async I/O)
-        let file_meta = match self.check_format_json().await {
-            Ok(meta) => meta,
-            Err(e) => {
-                // file does not exist or cannot be accessed, clear cached format info
-                if matches!(e, DiskError::UnformattedDisk | DiskError::DiskNotFound) {
-                    let mut format_info = self.format_info.write().await;
-                    format_info.id = None;
-                    format_info.file_info = None;
-                    format_info.data = Bytes::new();
-                    format_info.last_check = None;
-                }
-                return Err(e);
-            }
-        };
+        let file_meta = self.check_format_json().await?;
 
-        // Validate cache against current file metadata
-        if let (Some(cached_file_info), Some(id)) = (&file_info, id)
-            && super::fs::same_file(&file_meta, cached_file_info)
+        if let Some(file_info) = &format_info.file_info
+            && super::fs::same_file(&file_meta, file_info)
         {
-            // Cache is still valid, update last_check and return
             let mut format_info = self.format_info.write().await;
             format_info.last_check = Some(OffsetDateTime::now_utc());
-            return Ok(Some(id));
+            drop(format_info);
+
+            return Ok(id);
         }
 
         debug!("get_disk_id: read format.json");
@@ -1412,7 +1416,7 @@ impl DiskAPI for LocalDisk {
 
         let fm = FormatV3::try_from(b.as_slice()).map_err(|e| {
             warn!("decode format.json  err {:?}", e);
-            DiskError::UnformattedDisk
+            DiskError::CorruptedBackend
         })?;
 
         let (m, n) = fm.find_disk_index_by_disk_id(fm.erasure.this)?;
@@ -1958,6 +1962,7 @@ impl DiskAPI for LocalDisk {
             &opts,
             &mut out,
             &mut objs_returned,
+            false,
         )
         .await?;
 
@@ -2430,12 +2435,11 @@ impl DiskAPI for LocalDisk {
                     return self.write_metadata("", volume, path, fi).await;
                 }
 
-                let ret_err = if fi.version_id.is_some() {
-                    DiskError::FileVersionNotFound
+                return if fi.version_id.is_some() {
+                    Err(DiskError::FileVersionNotFound)
                 } else {
-                    DiskError::FileNotFound
+                    Err(DiskError::FileNotFound)
                 };
-                return Err(ret_err);
             }
         };
 
@@ -2654,6 +2658,56 @@ mod test {
         }
     }
 
+    #[tokio::test]
+    async fn test_scan_dir_includes_nested_object_dirs() {
+        use rustfs_filemeta::MetacacheReader;
+        use tempfile::tempdir;
+
+        let dir = tempdir().unwrap();
+        let bucket = "test-bucket";
+        let bucket_dir = dir.path().join(bucket);
+
+        fs::create_dir_all(bucket_dir.join("foo/bar/xyzzy")).await.unwrap();
+        fs::create_dir_all(bucket_dir.join("quux/thud")).await.unwrap();
+        fs::create_dir_all(bucket_dir.join("asdf")).await.unwrap();
+
+        fs::write(bucket_dir.join("foo/bar/xl.meta"), b"meta").await.unwrap();
+        fs::write(bucket_dir.join("foo/bar/xyzzy/xl.meta"), b"meta").await.unwrap();
+        fs::write(bucket_dir.join("quux/thud/xl.meta"), b"meta").await.unwrap();
+        fs::write(bucket_dir.join("asdf/xl.meta"), b"meta").await.unwrap();
+
+        let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
+        let disk = LocalDisk::new(&endpoint, false).await.unwrap();
+
+        let (reader, mut writer) = tokio::io::duplex(4096);
+        let mut out = MetacacheWriter::new(&mut writer);
+        let opts = WalkDirOptions {
+            bucket: bucket.to_string(),
+            base_dir: "".to_string(),
+            recursive: true,
+            ..Default::default()
+        };
+        let mut objs_returned = 0;
+
+        disk.scan_dir("".to_string(), "".to_string(), &opts, &mut out, &mut objs_returned, false)
+            .await
+            .unwrap();
+        out.close().await.unwrap();
+
+        let mut reader = MetacacheReader::new(reader);
+        let entries = reader.read_all().await.unwrap();
+        let names: Vec<String> = entries
+            .into_iter()
+            .filter(|entry| !entry.metadata.is_empty())
+            .map(|entry| entry.name)
+            .collect();
+
+        assert!(names.contains(&"asdf".to_string()));
+        assert!(names.contains(&"foo/bar".to_string()));
+        assert!(names.contains(&"foo/bar/xyzzy".to_string()));
+        assert!(names.contains(&"quux/thud".to_string()));
+    }
+
     #[tokio::test]
     async fn test_make_volume() {
         let p = "./testv0";

crates/ecstore/src/disk/mod.rs

@@ -412,6 +412,18 @@ impl DiskAPI for Disk {
     }
 }
 
+impl Disk {
+    /// Enable health monitoring on this disk.
+    /// Called after startup format loading completes so that remote peers
+    /// have time to come online before being marked as faulty.
+    pub fn enable_health_check(&self) {
+        match self {
+            Disk::Local(local_disk) => local_disk.enable_health_check(),
+            Disk::Remote(remote_disk) => remote_disk.enable_health_check(),
+        }
+    }
+}
+
 pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result<DiskStore> {
     if ep.is_local {
         let s = LocalDisk::new(ep, opt.cleanup).await?;

crates/ecstore/src/endpoints.rs

@@ -428,7 +428,7 @@ impl PoolEndpointList {
             }
         }
 
-        unimplemented!()
+        Ok(())
     }
 }
 

crates/ecstore/src/error.rs

@@ -20,186 +20,156 @@ use s3s::{S3Error, S3ErrorCode};
 pub type Error = StorageError;
 pub type Result<T> = core::result::Result<T, Error>;
 
+/// Storage layer error type covering disk, volume, bucket, object, multipart,
+/// erasure-coding, and operational error conditions.
+///
+/// Variants are organized by domain:
+///   - **Disk / Volume** – low-level storage I/O
+///   - **Bucket** – bucket-level operations
+///   - **Object** – object-level operations (including versioning)
+///   - **Multipart** – multipart upload lifecycle
+///   - **Erasure / Quorum** – erasure coding and quorum failures
+///   - **Operational** – decommission, healing, rate-limiting, etc.
+///   - **Generic** – I/O, locks, and catch-all errors
 #[derive(Debug, thiserror::Error)]
 pub enum StorageError {
+    // ── Disk / Volume ────────────────────────────────────────────────
     #[error("Faulty disk")]
     FaultyDisk,
-
-    #[error("Disk full")]
-    DiskFull,
-
-    #[error("Volume not found")]
-    VolumeNotFound,
-
-    #[error("Volume exists")]
-    VolumeExists,
-
-    #[error("File not found")]
-    FileNotFound,
-
-    #[error("File version not found")]
-    FileVersionNotFound,
-
-    #[error("File name too long")]
-    FileNameTooLong,
-
-    #[error("File access denied")]
-    FileAccessDenied,
-
-    #[error("File is corrupted")]
-    FileCorrupt,
-
-    #[error("Not a regular file")]
-    IsNotRegular,
-
-    #[error("Volume not empty")]
-    VolumeNotEmpty,
-
-    #[error("Volume access denied")]
-    VolumeAccessDenied,
-
-    #[error("Corrupted format")]
-    CorruptedFormat,
-
-    #[error("Corrupted backend")]
-    CorruptedBackend,
-
-    #[error("Unformatted disk")]
-    UnformattedDisk,
-
-    #[error("Disk not found")]
-    DiskNotFound,
-
-    #[error("Drive is root")]
-    DriveIsRoot,
-
     #[error("Faulty remote disk")]
     FaultyRemoteDisk,
-
+    #[error("Disk full")]
+    DiskFull,
+    #[error("Disk not found")]
+    DiskNotFound,
     #[error("Disk access denied")]
     DiskAccessDenied,
-
-    #[error("Unexpected error")]
-    Unexpected,
-
+    #[error("Drive is root")]
+    DriveIsRoot,
+    #[error("Unformatted disk")]
+    UnformattedDisk,
+    #[error("Corrupted format")]
+    CorruptedFormat,
+    #[error("Corrupted backend")]
+    CorruptedBackend,
     #[error("Too many open files")]
     TooManyOpenFiles,
 
-    #[error("No heal required")]
-    NoHealRequired,
+    #[error("Volume not found")]
+    VolumeNotFound,
+    #[error("Volume exists")]
+    VolumeExists,
+    #[error("Volume not empty")]
+    VolumeNotEmpty,
+    #[error("Volume access denied")]
+    VolumeAccessDenied,
 
-    #[error("Config not found")]
-    ConfigNotFound,
-
-    #[error("not implemented")]
-    NotImplemented,
-
-    #[error("Invalid arguments provided for {0}/{1}-{2}")]
-    InvalidArgument(String, String, String),
-
-    #[error("method not allowed")]
-    MethodNotAllowed,
+    #[error("File not found")]
+    FileNotFound,
+    #[error("File version not found")]
+    FileVersionNotFound,
+    #[error("File name too long")]
+    FileNameTooLong,
+    #[error("File access denied")]
+    FileAccessDenied,
+    #[error("File is corrupted")]
+    FileCorrupt,
+    #[error("Not a regular file")]
+    IsNotRegular,
 
+    // ── Bucket ───────────────────────────────────────────────────────
     #[error("Bucket not found: {0}")]
     BucketNotFound(String),
-
+    #[error("Bucket exists: {0}")]
+    BucketExists(String),
     #[error("Bucket not empty: {0}")]
     BucketNotEmpty(String),
-
     #[error("Bucket name invalid: {0}")]
     BucketNameInvalid(String),
 
-    #[error("Object name invalid: {0}/{1}")]
-    ObjectNameInvalid(String, String),
-
-    #[error("Bucket exists: {0}")]
-    BucketExists(String),
-    #[error("Storage reached its minimum free drive threshold.")]
-    StorageFull,
-    #[error("Please reduce your request rate")]
-    SlowDown,
-
-    #[error("Prefix access is denied:{0}/{1}")]
-    PrefixAccessDenied(String, String),
-
-    #[error("Invalid UploadID KeyCombination: {0}/{1}")]
-    InvalidUploadIDKeyCombination(String, String),
-
-    #[error("Malformed UploadID: {0}")]
-    MalformedUploadID(String),
-
-    #[error("Object name too long: {0}/{1}")]
-    ObjectNameTooLong(String, String),
-
-    #[error("Object name contains forward slash as prefix: {0}/{1}")]
-    ObjectNamePrefixAsSlash(String, String),
-
+    // ── Object ───────────────────────────────────────────────────────
     #[error("Object not found: {0}/{1}")]
     ObjectNotFound(String, String),
-
+    #[error("Object name invalid: {0}/{1}")]
+    ObjectNameInvalid(String, String),
+    #[error("Object name too long: {0}/{1}")]
+    ObjectNameTooLong(String, String),
+    #[error("Object name contains forward slash as prefix: {0}/{1}")]
+    ObjectNamePrefixAsSlash(String, String),
+    #[error("Object exists on :{0} as directory {1}")]
+    ObjectExistsAsDirectory(String, String),
     #[error("Version not found: {0}/{1}-{2}")]
     VersionNotFound(String, String, String),
-
-    #[error("Invalid upload id: {0}/{1}-{2}")]
-    InvalidUploadID(String, String, String),
-
-    #[error("Specified part could not be found. PartNumber {0}, Expected {1}, got {2}")]
-    InvalidPart(usize, String, String),
-
-    #[error("Your proposed upload is smaller than the minimum allowed size. Part {0} size {1} is less than minimum {2}")]
-    EntityTooSmall(usize, i64, i64),
-
     #[error("Invalid version id: {0}/{1}-{2}")]
     InvalidVersionID(String, String, String),
     #[error("invalid data movement operation, source and destination pool are the same for : {0}/{1}-{2}")]
     DataMovementOverwriteErr(String, String, String),
+    #[error("Prefix access is denied:{0}/{1}")]
+    PrefixAccessDenied(String, String),
 
-    #[error("Object exists on :{0} as directory {1}")]
-    ObjectExistsAsDirectory(String, String),
+    // ── Multipart ────────────────────────────────────────────────────
+    #[error("Invalid upload id: {0}/{1}-{2}")]
+    InvalidUploadID(String, String, String),
+    #[error("Invalid UploadID KeyCombination: {0}/{1}")]
+    InvalidUploadIDKeyCombination(String, String),
+    #[error("Malformed UploadID: {0}")]
+    MalformedUploadID(String),
+    #[error("Specified part could not be found. PartNumber {0}, Expected {1}, got {2}")]
+    InvalidPart(usize, String, String),
+    #[error("Invalid part number: {0}")]
+    InvalidPartNumber(usize),
+    #[error("Your proposed upload is smaller than the minimum allowed size. Part {0} size {1} is less than minimum {2}")]
+    EntityTooSmall(usize, i64, i64),
 
+    // ── Erasure / Quorum ─────────────────────────────────────────────
+    #[error("erasure read quorum")]
+    ErasureReadQuorum,
+    #[error("erasure write quorum")]
+    ErasureWriteQuorum,
     #[error("Storage resources are insufficient for the read operation: {0}/{1}")]
     InsufficientReadQuorum(String, String),
-
     #[error("Storage resources are insufficient for the write operation: {0}/{1}")]
     InsufficientWriteQuorum(String, String),
+    #[error("not first disk")]
+    NotFirstDisk,
+    #[error("first disk wait")]
+    FirstDiskWait,
 
+    // ── Operational ──────────────────────────────────────────────────
+    #[error("Storage reached its minimum free drive threshold.")]
+    StorageFull,
+    #[error("Please reduce your request rate")]
+    SlowDown,
     #[error("Decommission not started")]
     DecommissionNotStarted,
     #[error("Decommission already running")]
     DecommissionAlreadyRunning,
-
+    #[error("No heal required")]
+    NoHealRequired,
     #[error("DoneForNow")]
     DoneForNow,
-
-    #[error("erasure read quorum")]
-    ErasureReadQuorum,
-
-    #[error("erasure write quorum")]
-    ErasureWriteQuorum,
-
-    #[error("not first disk")]
-    NotFirstDisk,
-
-    #[error("first disk wait")]
-    FirstDiskWait,
-
-    #[error("Io error: {0}")]
-    Io(std::io::Error),
-
-    #[error("Lock error: {0}")]
-    Lock(#[from] rustfs_lock::LockError),
-
+    #[error("Config not found")]
+    ConfigNotFound,
     #[error("Precondition failed")]
     PreconditionFailed,
-
     #[error("Not modified")]
     NotModified,
-
-    #[error("Invalid part number: {0}")]
-    InvalidPartNumber(usize),
-
     #[error("Invalid range specified: {0}")]
     InvalidRangeSpec(String),
+
+    // ── Generic ──────────────────────────────────────────────────────
+    #[error("Unexpected error")]
+    Unexpected,
+    #[error("not implemented")]
+    NotImplemented,
+    #[error("Invalid arguments provided for {0}/{1}-{2}")]
+    InvalidArgument(String, String, String),
+    #[error("method not allowed")]
+    MethodNotAllowed,
+    #[error("Io error: {0}")]
+    Io(std::io::Error),
+    #[error("Lock error: {0}")]
+    Lock(#[from] rustfs_lock::LockError),
 }
 
 impl StorageError {
@@ -209,6 +179,29 @@ impl StorageError {
     {
         StorageError::Io(std::io::Error::other(error))
     }
+
+    pub fn is_not_found(&self) -> bool {
+        matches!(
+            self,
+            StorageError::FileNotFound
+                | StorageError::ObjectNotFound(_, _)
+                | StorageError::FileVersionNotFound
+                | StorageError::VersionNotFound(_, _, _)
+                | StorageError::VolumeNotFound
+                | StorageError::DiskNotFound
+                | StorageError::BucketNotFound(_)
+        )
+    }
+
+    pub fn is_quorum_error(&self) -> bool {
+        matches!(
+            self,
+            StorageError::ErasureReadQuorum
+                | StorageError::ErasureWriteQuorum
+                | StorageError::InsufficientReadQuorum(_, _)
+                | StorageError::InsufficientWriteQuorum(_, _)
+        )
+    }
 }
 
 impl From<DiskError> for StorageError {
@@ -724,24 +717,6 @@ pub fn is_all_volume_not_found(errs: &[Option<Error>]) -> bool {
     !errs.is_empty()
 }
 
-// pub fn is_all_not_found(errs: &[Option<Error>]) -> bool {
-//     for err in errs.iter() {
-//         if let Some(err) = err {
-//             if let Some(err) = err.downcast_ref::<DiskError>() {
-//                 match err {
-//                     DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => {
-//                         continue;
-//                     }
-//                     _ => return false,
-//                 }
-//             }
-//         }
-//         return false;
-//     }
-
-//     !errs.is_empty()
-// }
-
 pub fn to_object_err(err: Error, params: Vec<&str>) -> Error {
     match err {
         StorageError::DiskFull => StorageError::StorageFull,
@@ -938,10 +913,6 @@ pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error {
         object = decode_dir_object(params[1]);
     }
     match storage_err {
-        /*StorageError::NotImplemented => s3_error!(NotImplemented),
-        StorageError::InvalidArgument(bucket, object, version_id) => {
-            s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id)
-        }*/
         StorageError::MethodNotAllowed => S3Error::with_message(
             S3ErrorCode::MethodNotAllowed,
             ObjectApiError::MethodNotAllowed(GenericError {
@@ -951,73 +922,6 @@ pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error {
             })
             .to_string(),
         ),
-        /*StorageError::BucketNotFound(bucket) => {
-            s3_error!(NoSuchBucket, "bucket not found {}", bucket)
-        }
-        StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket),
-        StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket),
-        StorageError::ObjectNameInvalid(bucket, object) => {
-            s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object)
-        }
-        StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket),
-        StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."),
-        StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"),
-        StorageError::PrefixAccessDenied(bucket, object) => {
-            s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object)
-        }
-        StorageError::InvalidUploadIDKeyCombination(bucket, object) => {
-            s3_error!(InvalidArgument, "Invalid UploadID KeyCombination:  {}/{}", bucket, object)
-        }
-        StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket),
-        StorageError::ObjectNameTooLong(bucket, object) => {
-            s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object)
-        }
-        StorageError::ObjectNamePrefixAsSlash(bucket, object) => {
-            s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object)
-        }
-        StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object),
-        StorageError::VersionNotFound(bucket, object, version_id) => {
-            s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id)
-        }
-        StorageError::InvalidUploadID(bucket, object, version_id) => {
-            s3_error!(InvalidPart, "Invalid upload id:  {}/{}-{}", bucket, object, version_id)
-        }
-        StorageError::InvalidVersionID(bucket, object, version_id) => {
-            s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id)
-        }
-        // extended
-        StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!(
-            InvalidArgument,
-            "invalid data movement operation, source and destination pool are the same for : {}/{}-{}",
-            bucket,
-            object,
-            version_id
-        ),
-        // extended
-        StorageError::ObjectExistsAsDirectory(bucket, object) => {
-            s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object)
-        }
-        StorageError::InsufficientReadQuorum => {
-            s3_error!(SlowDown, "Storage resources are insufficient for the read operation")
-        }
-        StorageError::InsufficientWriteQuorum => {
-            s3_error!(SlowDown, "Storage resources are insufficient for the write operation")
-        }
-        StorageError::DecommissionNotStarted => s3_error!(InvalidArgument, "Decommission Not Started"),
-
-        StorageError::VolumeNotFound(bucket) => {
-            s3_error!(NoSuchBucket, "bucket not found {}", bucket)
-        }
-        StorageError::InvalidPart(bucket, object, version_id) => {
-            s3_error!(
-                InvalidPart,
-                "Specified part could not be found. PartNumber {}, Expected {}, got {}",
-                bucket,
-                object,
-                version_id
-            )
-        }
-        StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"),*/
         _ => s3s::S3Error::with_message(S3ErrorCode::Custom("err".into()), err.to_string()),
     }
 }

crates/ecstore/src/event/name.rs

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub enum EventName {
     ObjectAccessedGet,
     ObjectAccessedGetRetention,
@@ -61,11 +61,129 @@ pub enum EventName {
 
 impl EventName {
     fn expand(&self) -> Vec<EventName> {
-        todo!();
+        match self.clone() {
+            EventName::Everything => vec![
+                EventName::BucketCreated,
+                EventName::BucketRemoved,
+                EventName::ObjectAccessedAll,
+                EventName::ObjectCreatedAll,
+                EventName::ObjectRemovedAll,
+                EventName::ObjectManyVersions,
+                EventName::ObjectLargeVersions,
+                EventName::PrefixManyFolders,
+                EventName::ILMDelMarkerExpirationDelete,
+                EventName::ObjectReplicationAll,
+                EventName::ObjectRestoreAll,
+                EventName::ObjectTransitionAll,
+            ],
+            EventName::ObjectAccessedAll => vec![
+                EventName::ObjectAccessedGet,
+                EventName::ObjectAccessedGetRetention,
+                EventName::ObjectAccessedGetLegalHold,
+                EventName::ObjectAccessedHead,
+                EventName::ObjectAccessedAttributes,
+            ],
+            EventName::ObjectCreatedAll => vec![
+                EventName::ObjectCreatedCompleteMultipartUpload,
+                EventName::ObjectCreatedCopy,
+                EventName::ObjectCreatedPost,
+                EventName::ObjectCreatedPut,
+                EventName::ObjectCreatedPutRetention,
+                EventName::ObjectCreatedPutLegalHold,
+                EventName::ObjectCreatedPutTagging,
+                EventName::ObjectCreatedDeleteTagging,
+            ],
+            EventName::ObjectRemovedAll => vec![
+                EventName::ObjectRemovedDelete,
+                EventName::ObjectRemovedDeleteMarkerCreated,
+                EventName::ObjectRemovedNoOP,
+                EventName::ObjectRemovedDeleteAllVersions,
+            ],
+            EventName::ObjectReplicationAll => vec![
+                EventName::ObjectReplicationFailed,
+                EventName::ObjectReplicationComplete,
+                EventName::ObjectReplicationNotTracked,
+                EventName::ObjectReplicationMissedThreshold,
+                EventName::ObjectReplicationReplicatedAfterThreshold,
+            ],
+            EventName::ObjectRestoreAll => vec![EventName::ObjectRestorePost, EventName::ObjectRestoreCompleted],
+            EventName::ObjectTransitionAll => vec![EventName::ObjectTransitionFailed, EventName::ObjectTransitionComplete],
+            EventName::ObjectSingleTypesEnd | EventName::ObjectScannerAll => vec![self.clone()],
+            _ => vec![self.clone()],
+        }
     }
 
     fn mask(&self) -> u64 {
-        todo!();
+        match self {
+            EventName::Everything => u64::MAX,
+            EventName::BucketCreated => 1_u64 << 0,
+            EventName::BucketRemoved => 1_u64 << 1,
+            EventName::ObjectAccessedGet => 1_u64 << 2,
+            EventName::ObjectAccessedGetRetention => 1_u64 << 3,
+            EventName::ObjectAccessedGetLegalHold => 1_u64 << 4,
+            EventName::ObjectAccessedHead => 1_u64 << 5,
+            EventName::ObjectAccessedAttributes => 1_u64 << 6,
+            EventName::ObjectCreatedCompleteMultipartUpload => 1_u64 << 7,
+            EventName::ObjectCreatedCopy => 1_u64 << 8,
+            EventName::ObjectCreatedPost => 1_u64 << 9,
+            EventName::ObjectCreatedPut => 1_u64 << 10,
+            EventName::ObjectCreatedPutRetention => 1_u64 << 11,
+            EventName::ObjectCreatedPutLegalHold => 1_u64 << 12,
+            EventName::ObjectCreatedPutTagging => 1_u64 << 13,
+            EventName::ObjectCreatedDeleteTagging => 1_u64 << 14,
+            EventName::ObjectRemovedDelete => 1_u64 << 15,
+            EventName::ObjectRemovedDeleteMarkerCreated => 1_u64 << 16,
+            EventName::ObjectRemovedDeleteAllVersions => 1_u64 << 17,
+            EventName::ObjectRemovedNoOP => 1_u64 << 18,
+            EventName::ObjectManyVersions => 1_u64 << 19,
+            EventName::ObjectLargeVersions => 1_u64 << 20,
+            EventName::PrefixManyFolders => 1_u64 << 21,
+            EventName::ILMDelMarkerExpirationDelete => 1_u64 << 22,
+            EventName::ObjectReplicationFailed => 1_u64 << 23,
+            EventName::ObjectReplicationComplete => 1_u64 << 24,
+            EventName::ObjectReplicationMissedThreshold => 1_u64 << 25,
+            EventName::ObjectReplicationReplicatedAfterThreshold => 1_u64 << 26,
+            EventName::ObjectReplicationNotTracked => 1_u64 << 27,
+            EventName::ObjectRestorePost => 1_u64 << 28,
+            EventName::ObjectRestoreCompleted => 1_u64 << 29,
+            EventName::ObjectRestoreAll => 1_u64 << 30,
+            EventName::ObjectTransitionFailed => 1_u64 << 31,
+            EventName::ObjectTransitionComplete => 1_u64 << 32,
+            EventName::ObjectAccessedAll => {
+                EventName::ObjectAccessedGet.mask()
+                    | EventName::ObjectAccessedGetRetention.mask()
+                    | EventName::ObjectAccessedGetLegalHold.mask()
+                    | EventName::ObjectAccessedHead.mask()
+                    | EventName::ObjectAccessedAttributes.mask()
+            }
+            EventName::ObjectCreatedAll => {
+                EventName::ObjectCreatedCompleteMultipartUpload.mask()
+                    | EventName::ObjectCreatedCopy.mask()
+                    | EventName::ObjectCreatedPost.mask()
+                    | EventName::ObjectCreatedPut.mask()
+                    | EventName::ObjectCreatedPutRetention.mask()
+                    | EventName::ObjectCreatedPutLegalHold.mask()
+                    | EventName::ObjectCreatedPutTagging.mask()
+                    | EventName::ObjectCreatedDeleteTagging.mask()
+            }
+            EventName::ObjectRemovedAll => {
+                EventName::ObjectRemovedDelete.mask()
+                    | EventName::ObjectRemovedDeleteMarkerCreated.mask()
+                    | EventName::ObjectRemovedNoOP.mask()
+                    | EventName::ObjectRemovedDeleteAllVersions.mask()
+            }
+            EventName::ObjectReplicationAll => {
+                EventName::ObjectReplicationFailed.mask()
+                    | EventName::ObjectReplicationComplete.mask()
+                    | EventName::ObjectReplicationMissedThreshold.mask()
+                    | EventName::ObjectReplicationReplicatedAfterThreshold.mask()
+                    | EventName::ObjectReplicationNotTracked.mask()
+            }
+            EventName::ObjectTransitionAll => {
+                EventName::ObjectTransitionFailed.mask() | EventName::ObjectTransitionComplete.mask()
+            }
+            EventName::ObjectSingleTypesEnd | EventName::ObjectScannerAll => 0,
+        }
     }
 }
 

crates/ecstore/src/event_notification.rs

@@ -21,7 +21,9 @@ use crate::store::ECStore;
 use crate::store_api::ObjectInfo;
 use std::collections::HashMap;
 use std::sync::Arc;
+use std::sync::atomic::Ordering;
 use tokio::sync::RwLock;
+use tracing::warn;
 
 pub struct EventNotifier {
     target_list: TargetList,
@@ -37,24 +39,33 @@ impl EventNotifier {
     }
 
     fn get_arn_list(&self) -> Vec<String> {
-        todo!();
+        warn!(
+            event_count = self.target_list.total_events.load(Ordering::Relaxed),
+            "event notifier arn list requested but not implemented in this build"
+        );
+        Vec::new()
     }
 
     fn set(&self, bucket: &str, meta: BucketMetadata) {
-        todo!();
+        warn!(bucket = bucket, "event notifier set() called but currently no-op in this build");
     }
 
-    fn init_bucket_targets(&self, api: ECStore) -> Result<(), std::io::Error> {
+    fn init_bucket_targets(&self, _api: ECStore) -> Result<(), std::io::Error> {
         /*if err := self.target_list.Add(globalNotifyTargetList.Targets()...); err != nil {
           return err
         }
         self.target_list = self.target_list.Init(runtime.GOMAXPROCS(0)) // TODO: make this configurable (y4m4)
         nil*/
-        todo!();
+        warn!("init_bucket_targets called but currently no-op in this build");
+        Ok(())
     }
 
     fn send(&self, args: EventArgs) {
-        todo!();
+        warn!(
+            event_name = args.event_name,
+            bucket = args.bucket_name,
+            "event send() called but notifier is not fully implemented in this build"
+        );
     }
 }
 

crates/ecstore/src/global.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::bucket::bandwidth::monitor::Monitor;
 use crate::{
     bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys,
     disk::DiskStore,
@@ -29,6 +30,7 @@ use std::{
 };
 use tokio::sync::{OnceCell, RwLock};
 use tokio_util::sync::CancellationToken;
+use tracing::warn;
 use uuid::Uuid;
 
 pub const DISK_ASSUME_UNKNOWN_SIZE: u64 = 1 << 30;
@@ -40,7 +42,6 @@ lazy_static! {
     static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
     static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
     pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
-    pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
     pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
     pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
     pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
@@ -55,9 +56,23 @@ lazy_static! {
     pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
     pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
     pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();
-    pub static ref GLOBAL_REGION: OnceLock<String> = OnceLock::new();
-    pub static ref GLOBAL_LOCAL_LOCK_CLIENT: OnceLock<Arc<dyn rustfs_lock::client::LockClient>> = OnceLock::new();
+    pub static ref GLOBAL_REGION: OnceLock<s3s::region::Region> = OnceLock::new();
+    pub static ref GLOBAL_LOCAL_LOCK_CLIENT: OnceLock<Arc<dyn LockClient>> = OnceLock::new();
     pub static ref GLOBAL_LOCK_CLIENTS: OnceLock<HashMap<String, Arc<dyn LockClient>>> = OnceLock::new();
+    pub static ref GLOBAL_BUCKET_MONITOR: OnceLock<Arc<Monitor>> = OnceLock::new();
+}
+
+pub fn init_global_bucket_monitor(num_nodes: u64) {
+    if GLOBAL_BUCKET_MONITOR.set(Monitor::new(num_nodes)).is_err() {
+        warn!(
+            "global bucket monitor already initialized, ignoring re-initialization with num_nodes={}",
+            num_nodes
+        );
+    }
+}
+
+pub fn get_global_bucket_monitor() -> Option<Arc<Monitor>> {
+    GLOBAL_BUCKET_MONITOR.get().cloned()
 }
 
 /// Global cancellation token for background services (data scanner and auto heal)
@@ -134,6 +149,14 @@ pub fn get_global_endpoints() -> EndpointServerPools {
     }
 }
 
+pub fn get_global_endpoints_opt() -> Option<EndpointServerPools> {
+    GLOBAL_Endpoints.get().cloned()
+}
+
+pub fn get_global_tier_config_mgr() -> Arc<RwLock<TierConfigMgr>> {
+    GLOBAL_TierConfigMgr.clone()
+}
+
 /// Create a new object layer instance
 ///
 /// # Returns
@@ -219,20 +242,20 @@ type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;
 /// Set the global region
 ///
 /// # Arguments
-/// * `region` - The region string to set globally
+/// * `region` - The Region instance to set globally
 ///
 /// # Returns
 /// * None
-pub fn set_global_region(region: String) {
+pub fn set_global_region(region: s3s::region::Region) {
     GLOBAL_REGION.set(region).unwrap();
 }
 
 /// Get the global region
 ///
 /// # Returns
-/// * `Option<String>` - The global region string, if set
+/// * `Option<s3s::region::Region>` - The global region, if set
 ///
-pub fn get_global_region() -> Option<String> {
+pub fn get_global_region() -> Option<s3s::region::Region> {
     GLOBAL_REGION.get().cloned()
 }