Compare commits

..

1 Commits

Author SHA1 Message Date
copilot-swe-agent[bot] fe637d2087 Initial plan 2025-11-06 13:30:27 +00:00
721 changed files with 41852 additions and 78051 deletions
-64
View File
@@ -1,64 +0,0 @@
## —— Development/Source builds using direct buildx commands ---------------------------------------
.PHONY: docker-dev
docker-dev: ## Build dev multi-arch image (cannot load locally)
@echo "🏗️ Building multi-architecture development Docker images with buildx..."
@echo "💡 This builds from source code and is intended for local development and testing"
@echo "⚠️ Multi-arch images cannot be loaded locally, use docker-dev-push to push to registry"
$(DOCKER_CLI) buildx build \
--platform linux/amd64,linux/arm64 \
--file $(DOCKERFILE_SOURCE) \
--tag rustfs:source-latest \
--tag rustfs:dev-latest \
.
.PHONY: docker-dev-local
docker-dev-local: ## Build dev single-arch image (local load)
@echo "🏗️ Building single-architecture development Docker image for local use..."
@echo "💡 This builds from source code for the current platform and loads locally"
$(DOCKER_CLI) buildx build \
--file $(DOCKERFILE_SOURCE) \
--tag rustfs:source-latest \
--tag rustfs:dev-latest \
--load \
.
.PHONY: docker-dev-push
docker-dev-push: ## Build and push multi-arch development image # e.g (make docker-dev-push REGISTRY=xxx)
@if [ -z "$(REGISTRY)" ]; then \
echo "❌ Error: Please specify registry, example: make docker-dev-push REGISTRY=ghcr.io/username"; \
exit 1; \
fi
@echo "🚀 Building and pushing multi-architecture development Docker images..."
@echo "💡 Pushing to registry: $(REGISTRY)"
$(DOCKER_CLI) buildx build \
--platform linux/amd64,linux/arm64 \
--file $(DOCKERFILE_SOURCE) \
--tag $(REGISTRY)/rustfs:source-latest \
--tag $(REGISTRY)/rustfs:dev-latest \
--push \
.
.PHONY: dev-env-start
dev-env-start: ## Start development container environment
@echo "🚀 Starting development environment..."
$(DOCKER_CLI) buildx build \
--file $(DOCKERFILE_SOURCE) \
--tag rustfs:dev \
--load \
.
$(DOCKER_CLI) stop $(CONTAINER_NAME) 2>/dev/null || true
$(DOCKER_CLI) rm $(CONTAINER_NAME) 2>/dev/null || true
$(DOCKER_CLI) run -d --name $(CONTAINER_NAME) \
-p 9010:9010 -p 9000:9000 \
-v $(shell pwd):/workspace \
-it rustfs:dev
.PHONY: dev-env-stop
dev-env-stop: ## Stop development container environment
@echo "🛑 Stopping development environment..."
$(DOCKER_CLI) stop $(CONTAINER_NAME) 2>/dev/null || true
$(DOCKER_CLI) rm $(CONTAINER_NAME) 2>/dev/null || true
.PHONY: dev-env-restart
dev-env-restart: dev-env-stop dev-env-start ## Restart development container environment
@@ -1,41 +0,0 @@
## —— Production builds using docker buildx (for CI/CD and production) -----------------------------
.PHONY: docker-buildx
docker-buildx: ## Build production multi-arch image (no push)
@echo "🏗️ Building multi-architecture production Docker images with buildx..."
./docker-buildx.sh
.PHONY: docker-buildx-push
docker-buildx-push: ## Build and push production multi-arch image
@echo "🚀 Building and pushing multi-architecture production Docker images with buildx..."
./docker-buildx.sh --push
.PHONY: docker-buildx-version
docker-buildx-version: ## Build and version production multi-arch image # e.g (make docker-buildx-version VERSION=v1.0.0)
@if [ -z "$(VERSION)" ]; then \
echo "❌ Error: Please specify version, example: make docker-buildx-version VERSION=v1.0.0"; \
exit 1; \
fi
@echo "🏗️ Building multi-architecture production Docker images (version: $(VERSION))..."
./docker-buildx.sh --release $(VERSION)
.PHONY: docker-buildx-push-version
docker-buildx-push-version: ## Build and version and push production multi-arch image # e.g (make docker-buildx-push-version VERSION=v1.0.0)
@if [ -z "$(VERSION)" ]; then \
echo "❌ Error: Please specify version, example: make docker-buildx-push-version VERSION=v1.0.0"; \
exit 1; \
fi
@echo "🚀 Building and pushing multi-architecture production Docker images (version: $(VERSION))..."
./docker-buildx.sh --release $(VERSION) --push
.PHONY: docker-buildx-production-local
docker-buildx-production-local: ## Build production single-arch image locally
@echo "🏗️ Building single-architecture production Docker image locally..."
@echo "💡 Alternative to docker-buildx.sh for local testing"
$(DOCKER_CLI) buildx build \
--file $(DOCKERFILE_PRODUCTION) \
--tag rustfs:production-latest \
--tag rustfs:latest \
--load \
--build-arg RELEASE=latest \
.
-16
View File
@@ -1,16 +0,0 @@
## —— Single Architecture Docker Builds (Traditional) ----------------------------------------------
.PHONY: docker-build-production
docker-build-production: ## Build single-arch production image
@echo "🏗️ Building single-architecture production Docker image..."
@echo "💡 Consider using 'make docker-buildx-production-local' for multi-arch support"
$(DOCKER_CLI) build -f $(DOCKERFILE_PRODUCTION) -t rustfs:latest .
.PHONY: docker-build-source
docker-build-source: ## Build single-arch source image
@echo "🏗️ Building single-architecture source Docker image..."
@echo "💡 Consider using 'make docker-dev-local' for multi-arch support"
DOCKER_BUILDKIT=1 $(DOCKER_CLI) build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
-f $(DOCKERFILE_SOURCE) -t rustfs:source .
-22
View File
@@ -1,22 +0,0 @@
## —— Docker-based build (alternative approach) ----------------------------------------------------
# Usage: make BUILD_OS=ubuntu22.04 build-docker
# Output: target/ubuntu22.04/release/rustfs
.PHONY: build-docker
build-docker: SOURCE_BUILD_IMAGE_NAME = rustfs-$(BUILD_OS):v1
build-docker: SOURCE_BUILD_CONTAINER_NAME = rustfs-$(BUILD_OS)-build
build-docker: BUILD_CMD = /root/.cargo/bin/cargo build --release --bin rustfs --target-dir /root/s3-rustfs/target/$(BUILD_OS)
build-docker: ## Build using Docker container # e.g (make build-docker BUILD_OS=ubuntu22.04)
@echo "🐳 Building RustFS using Docker ($(BUILD_OS))..."
$(DOCKER_CLI) buildx build -t $(SOURCE_BUILD_IMAGE_NAME) -f $(DOCKERFILE_SOURCE) .
$(DOCKER_CLI) run --rm --name $(SOURCE_BUILD_CONTAINER_NAME) -v $(shell pwd):/root/s3-rustfs -it $(SOURCE_BUILD_IMAGE_NAME) $(BUILD_CMD)
.PHONY: docker-inspect-multiarch
docker-inspect-multiarch: ## Check image architecture support
@if [ -z "$(IMAGE)" ]; then \
echo "❌ Error: Please specify image, example: make docker-inspect-multiarch IMAGE=rustfs/rustfs:latest"; \
exit 1; \
fi
@echo "🔍 Inspecting multi-architecture image: $(IMAGE)"
docker buildx imagetools inspect $(IMAGE)
-55
View File
@@ -1,55 +0,0 @@
## —— Local Native Build using build-rustfs.sh script (Recommended) --------------------------------
.PHONY: build
build: ## Build RustFS binary (includes console by default)
@echo "🔨 Building RustFS using build-rustfs.sh script..."
./build-rustfs.sh
.PHONY: build-dev
build-dev: ## Build RustFS in Development mode
@echo "🔨 Building RustFS in development mode..."
./build-rustfs.sh --dev
.PHONY: build-musl
build-musl: ## Build x86_64 musl version
@echo "🔨 Building rustfs for x86_64-unknown-linux-musl..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform x86_64-unknown-linux-musl
.PHONY: build-gnu
build-gnu: ## Build x86_64 GNU version
@echo "🔨 Building rustfs for x86_64-unknown-linux-gnu..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform x86_64-unknown-linux-gnu
.PHONY: build-musl-arm64
build-musl-arm64: ## Build aarch64 musl version
@echo "🔨 Building rustfs for aarch64-unknown-linux-musl..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform aarch64-unknown-linux-musl
.PHONY: build-gnu-arm64
build-gnu-arm64: ## Build aarch64 GNU version
@echo "🔨 Building rustfs for aarch64-unknown-linux-gnu..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform aarch64-unknown-linux-gnu
.PHONY: build-cross-all
build-cross-all: core-deps ## Build binaries for all architectures
@echo "🔧 Building all target architectures..."
@echo "💡 On macOS/Windows, use 'make docker-dev' for reliable multi-arch builds"
@echo "🔨 Generating protobuf code..."
cargo run --bin gproto || true
@echo "🔨 Building rustfs for x86_64-unknown-linux-musl..."
./build-rustfs.sh --platform x86_64-unknown-linux-musl
@echo "🔨 Building rustfs for x86_64-unknown-linux-gnu..."
./build-rustfs.sh --platform x86_64-unknown-linux-gnu
@echo "🔨 Building rustfs for aarch64-unknown-linux-musl..."
./build-rustfs.sh --platform aarch64-unknown-linux-musl
@echo "🔨 Building rustfs for aarch64-unknown-linux-gnu..."
./build-rustfs.sh --platform aarch64-unknown-linux-gnu
-24
View File
@@ -1,24 +0,0 @@
## —— Check and Inform Dependencies ----------------------------------------------------------------
# Fatal check
# Checks all required dependencies and exits with error if not found
# (e.g., cargo, rustfmt)
check-%:
@command -v $* >/dev/null 2>&1 || { \
echo >&2 "❌ '$*' is not installed."; \
exit 1; \
}
# Warning-only check
# Checks for optional dependencies and issues a warning if not found
# (e.g., cargo-nextest for enhanced testing)
warn-%:
@command -v $* >/dev/null 2>&1 || { \
echo >&2 "⚠️ '$*' is not installed."; \
}
# For checking dependencies use check-<dep-name> or warn-<dep-name>
.PHONY: core-deps fmt-deps test-deps
core-deps: check-cargo ## Check core dependencies
fmt-deps: check-rustfmt ## Check lint and formatting dependencies
test-deps: warn-cargo-nextest ## Check tests dependencies
-6
View File
@@ -1,6 +0,0 @@
## —— Deploy using dev_deploy.sh script ------------------------------------------------------------
.PHONY: deploy-dev
deploy-dev: build-musl ## Deploy to dev server
@echo "🚀 Deploying to dev server: $${IP}"
./scripts/dev_deploy.sh $${IP}
-38
View File
@@ -1,38 +0,0 @@
## —— Help, Help Build and Help Docker -------------------------------------------------------------
.PHONY: help
help: ## Shows This Help Menu
echo -e "$$HEADER"
grep -E '(^[a-zA-Z0-9_-]+:.*?## .*$$)|(^## )' $(MAKEFILE_LIST) | sed 's/^[^:]*://g' | awk 'BEGIN {FS = ":.*?## | #"} ; {printf "${cyan}%-30s${reset} ${white}%s${reset} ${green}%s${reset}\n", $$1, $$2, $$3}' | sed -e 's/\[36m##/\n[32m##/'
.PHONY: help-build
help-build: ## Shows RustFS build help
@echo ""
@echo "💡 build-rustfs.sh script provides more options, smart detection and binary verification"
@echo ""
@echo "🔧 Direct usage of build-rustfs.sh script:"
@echo ""
@echo " ./build-rustfs.sh --help # View script help"
@echo " ./build-rustfs.sh --no-console # Build without console resources"
@echo " ./build-rustfs.sh --force-console-update # Force update console resources"
@echo " ./build-rustfs.sh --dev # Development mode build"
@echo " ./build-rustfs.sh --sign # Sign binary files"
@echo " ./build-rustfs.sh --platform x86_64-unknown-linux-gnu # Specify target platform"
@echo " ./build-rustfs.sh --skip-verification # Skip binary verification"
@echo ""
.PHONY: help-docker
help-docker: ## Shows docker environment and suggestion help
@echo ""
@echo "📋 Environment Variables:"
@echo " REGISTRY Image registry address (required for push)"
@echo " DOCKERHUB_USERNAME Docker Hub username"
@echo " DOCKERHUB_TOKEN Docker Hub access token"
@echo " GITHUB_TOKEN GitHub access token"
@echo ""
@echo "💡 Suggestions:"
@echo " Production use: Use docker-buildx* commands (based on precompiled binaries)"
@echo " Local development: Use docker-dev* commands (build from source)"
@echo " Development environment: Use dev-env-* commands to manage dev containers"
@echo ""
-22
View File
@@ -1,22 +0,0 @@
## —— Code quality and Formatting ------------------------------------------------------------------
.PHONY: fmt
fmt: core-deps fmt-deps ## Format code
@echo "🔧 Formatting code..."
cargo fmt --all
.PHONY: fmt-check
fmt-check: core-deps fmt-deps ## Check code formatting
@echo "📝 Checking code formatting..."
cargo fmt --all --check
.PHONY: clippy-check
clippy-check: core-deps ## Run clippy checks
@echo "🔍 Running clippy checks..."
cargo clippy --fix --allow-dirty
cargo clippy --all-targets --all-features -- -D warnings
.PHONY: compilation-check
compilation-check: core-deps ## Run compilation check
@echo "🔨 Running compilation check..."
cargo check --all-targets
-11
View File
@@ -1,11 +0,0 @@
## —— Pre Commit Checks ----------------------------------------------------------------------------
.PHONY: setup-hooks
setup-hooks: ## Set up git hooks
@echo "🔧 Setting up git hooks..."
chmod +x .git/hooks/pre-commit
@echo "✅ Git hooks setup complete!"
.PHONY: pre-commit
pre-commit: fmt clippy-check compilation-check test ## Run pre-commit checks
@echo "✅ All pre-commit checks passed!"
-20
View File
@@ -1,20 +0,0 @@
## —— Tests and e2e test ---------------------------------------------------------------------------
.PHONY: test
test: core-deps test-deps ## Run all tests
@echo "🧪 Running tests..."
@if command -v cargo-nextest >/dev/null 2>&1; then \
cargo nextest run --all --exclude e2e_test; \
else \
echo "️ cargo-nextest not found; falling back to 'cargo test'"; \
cargo test --workspace --exclude e2e_test -- --nocapture; \
fi
cargo test --all --doc
.PHONY: e2e-server
e2e-server: ## Run e2e-server tests
sh $(shell pwd)/scripts/run.sh
.PHONY: probe-e2e
probe-e2e: ## Probe e2e tests
sh $(shell pwd)/scripts/probe.sh
@@ -69,9 +69,6 @@ services:
- TZ=Asia/Shanghai
networks:
- rustfs-network
volumes:
- ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro
- ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
node1:
build:
@@ -82,7 +79,7 @@ services:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
- RUSTFS_OBS_LOGGER_LEVEL=debug
platform: linux/amd64
ports:
@@ -99,7 +96,7 @@ services:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
- RUSTFS_OBS_LOGGER_LEVEL=debug
platform: linux/amd64
ports:
@@ -116,7 +113,7 @@ services:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
- RUSTFS_OBS_LOGGER_LEVEL=debug
platform: linux/amd64
ports:
@@ -133,7 +130,7 @@ services:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
- RUSTFS_OBS_LOGGER_LEVEL=debug
platform: linux/amd64
ports:
+1 -1
View File
@@ -56,7 +56,7 @@ docker compose -f docker-compose.yml -f docker-compose.override.yml up -d
- docker compose v2.0.0 or before
```bash
docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d
docke-compose -f docker-compose.yml -f docker-compose.override.yml up -d
```
To access the Grafana dashboard, navigate to `http://localhost:3000` in your browser. The default username and password
+19 -91
View File
@@ -16,7 +16,7 @@ services:
tempo-init:
image: busybox:latest
command: [ "sh", "-c", "chown -R 10001:10001 /var/tempo" ]
command: ["sh", "-c", "chown -R 10001:10001 /var/tempo"]
volumes:
- ./tempo-data:/var/tempo
user: root
@@ -34,145 +34,73 @@ services:
ports:
- "3200:3200" # tempo
- "24317:4317" # otlp grpc
- "24318:4318" # otlp http
restart: unless-stopped
networks:
- otel-network
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/metrics" ]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
otel-collector:
image: otel/opentelemetry-collector-contrib:latest
image: otel/opentelemetry-collector-contrib:0.129.1
environment:
- TZ=Asia/Shanghai
volumes:
- ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
- ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
ports:
- "1888:1888" # pprof
- "8888:8888" # Prometheus metrics for Collector
- "8889:8889" # Prometheus metrics for application indicators
- "13133:13133" # health check
- "4317:4317" # OTLP gRPC
- "4318:4318" # OTLP HTTP
- "55679:55679" # zpages
- "1888:1888"
- "8888:8888"
- "8889:8889"
- "13133:13133"
- "4317:4317"
- "4318:4318"
- "55679:55679"
networks:
- otel-network
depends_on:
jaeger:
condition: service_started
tempo:
condition: service_started
prometheus:
condition: service_started
loki:
condition: service_started
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
interval: 10s
timeout: 5s
retries: 3
jaeger:
image: jaegertracing/jaeger:latest
image: jaegertracing/jaeger:2.8.0
environment:
- TZ=Asia/Shanghai
- SPAN_STORAGE_TYPE=memory
- COLLECTOR_OTLP_ENABLED=true
ports:
- "16686:16686" # Web UI
- "14317:4317" # OTLP gRPC
- "14318:4318" # OTLP HTTP
- "18888:8888" # collector
- "16686:16686"
- "14317:4317"
- "14318:4318"
networks:
- otel-network
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:16686" ]
interval: 10s
timeout: 5s
retries: 3
prometheus:
image: prom/prometheus:latest
image: prom/prometheus:v3.4.2
environment:
- TZ=Asia/Shanghai
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./prometheus-data:/prometheus
- ./prometheus.yml:/etc/prometheus/prometheus.yml
ports:
- "9090:9090"
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--web.enable-otlp-receiver' # Enable OTLP
- '--web.enable-remote-write-receiver' # Enable remote write
- '--enable-feature=promql-experimental-functions' # Enable info()
- '--storage.tsdb.min-block-duration=15m' # Minimum block duration
- '--storage.tsdb.max-block-duration=1h' # Maximum block duration
- '--log.level=info'
- '--storage.tsdb.retention.time=30d'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
- '--web.console.templates=/usr/share/prometheus/consoles'
restart: unless-stopped
networks:
- otel-network
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
interval: 10s
timeout: 5s
retries: 3
loki:
image: grafana/loki:latest
image: grafana/loki:3.5.1
environment:
- TZ=Asia/Shanghai
volumes:
- ./loki-config.yaml:/etc/loki/local-config.yaml:ro
- ./loki-config.yaml:/etc/loki/local-config.yaml
ports:
- "3100:3100"
command: -config.file=/etc/loki/local-config.yaml
networks:
- otel-network
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
interval: 10s
timeout: 5s
retries: 3
grafana:
image: grafana/grafana:latest
image: grafana/grafana:12.0.2
ports:
- "3000:3000" # Web UI
volumes:
- ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
environment:
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_SECURITY_ADMIN_USER=admin
- TZ=Asia/Shanghai
- GF_INSTALL_PLUGINS=grafana-pyroscope-datasource
restart: unless-stopped
networks:
- otel-network
depends_on:
- prometheus
- tempo
- loki
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
interval: 10s
timeout: 5s
retries: 3
volumes:
prometheus-data:
tempo-data:
networks:
otel-network:
driver: bridge
name: "network_otel_config"
ipam:
config:
- subnet: 172.28.0.0/16
driver_opts:
com.docker.network.enable_ipv6: "true"
@@ -0,0 +1,32 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
orgId: 1
url: http://prometheus:9090
basicAuth: false
isDefault: false
version: 1
editable: false
jsonData:
httpMethod: GET
- name: Tempo
type: tempo
access: proxy
orgId: 1
url: http://tempo:3200
basicAuth: false
isDefault: true
version: 1
editable: false
apiVersion: 1
uid: tempo
jsonData:
httpMethod: GET
serviceMap:
datasourceUid: prometheus
streamingEnabled:
search: true
File diff suppressed because it is too large Load Diff
@@ -1,108 +0,0 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
orgId: 1
url: http://prometheus:9090
basicAuth: false
isDefault: false
version: 1
editable: false
jsonData:
httpMethod: GET
- name: Tempo
type: tempo
access: proxy
orgId: 1
url: http://tempo:3200
basicAuth: false
isDefault: true
version: 1
editable: false
apiVersion: 1
uid: tempo
jsonData:
httpMethod: GET
serviceMap:
datasourceUid: prometheus
streamingEnabled:
search: true
tracesToLogsV2:
# Field with an internal link pointing to a logs data source in Grafana.
# datasourceUid value must match the uid value of the logs data source.
datasourceUid: 'loki'
spanStartTimeShift: '-1h'
spanEndTimeShift: '1h'
tags: [ 'job', 'instance', 'pod', 'namespace' ]
filterByTraceID: false
filterBySpanID: false
customQuery: true
query: 'method="$${__span.tags.method}"'
tracesToMetrics:
datasourceUid: 'prometheus'
spanStartTimeShift: '-1h'
spanEndTimeShift: '1h'
tags: [ { key: 'service.name', value: 'service' }, { key: 'job' } ]
queries:
- name: 'Sample query'
query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
tracesToProfiles:
datasourceUid: 'grafana-pyroscope-datasource'
tags: [ 'job', 'instance', 'pod', 'namespace' ]
profileTypeId: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds'
customQuery: true
query: 'method="$${__span.tags.method}"'
serviceMap:
datasourceUid: 'prometheus'
nodeGraph:
enabled: true
search:
hide: false
traceQuery:
timeShiftEnabled: true
spanStartTimeShift: '-1h'
spanEndTimeShift: '1h'
spanBar:
type: 'Tag'
tag: 'http.path'
streamingEnabled:
search: true
- name: Jaeger
type: jaeger
uid: Jaeger
url: http://jaeger:16686
basicAuth: false
access: proxy
readOnly: false
isDefault: false
jsonData:
tracesToLogsV2:
# Field with an internal link pointing to a logs data source in Grafana.
# datasourceUid value must match the uid value of the logs data source.
datasourceUid: 'loki'
spanStartTimeShift: '1h'
spanEndTimeShift: '-1h'
tags: [ 'job', 'instance', 'pod', 'namespace' ]
filterByTraceID: false
filterBySpanID: false
customQuery: true
query: 'method="$${__span.tags.method}"'
tracesToMetrics:
datasourceUid: 'Prometheus'
spanStartTimeShift: '1h'
spanEndTimeShift: '-1h'
tags: [ { key: 'service.name', value: 'service' }, { key: 'job' } ]
queries:
- name: 'Sample query'
query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
nodeGraph:
enabled: true
traceQuery:
timeShiftEnabled: true
spanStartTimeShift: '1h'
spanEndTimeShift: '-1h'
spanBar:
type: 'None'
-2
View File
@@ -65,7 +65,6 @@ extensions:
some_store:
memory:
max_traces: 1000000
max_events: 100000
another_store:
memory:
max_traces: 1000000
@@ -103,7 +102,6 @@ receivers:
processors:
batch:
metadata_keys: [ "span.kind", "http.method", "http.status_code", "db.system", "db.statement", "messaging.system", "messaging.destination", "messaging.operation","span.events","span.links" ]
# Adaptive Sampling Processor is required to support adaptive sampling.
# It expects remote_sampling extension with `adaptive:` config to be enabled.
adaptive_sampling:
-5
View File
@@ -41,9 +41,6 @@ query_range:
limits_config:
metric_aggregation_enabled: true
max_line_size: 256KB
max_line_size_truncate: false
allow_structured_metadata: true
schema_config:
configs:
@@ -54,7 +51,6 @@ schema_config:
index:
prefix: index_
period: 24h
row_shards: 16
pattern_ingester:
enabled: true
@@ -67,7 +63,6 @@ ruler:
frontend:
encoding: protobuf
# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
#
@@ -15,107 +15,67 @@
receivers:
otlp:
protocols:
grpc: # OTLP gRPC receiver
grpc: # OTLP gRPC 接收器
endpoint: 0.0.0.0:4317
http: # OTLP HTTP receiver
http: # OTLP HTTP 接收器
endpoint: 0.0.0.0:4318
processors:
batch: # Batch processor to improve throughput
batch: # 批处理处理器,提升吞吐量
timeout: 5s
send_batch_size: 1000
metadata_keys: [ ]
metadata_cardinality_limit: 1000
memory_limiter:
check_interval: 1s
limit_mib: 512
transform/logs:
log_statements:
- context: log
statements:
# Extract Body as attribute "message"
- set(attributes["message"], body.string)
# Retain the original Body
- set(attributes["log.body"], body.string)
exporters:
otlp/traces: # OTLP exporter for trace data
endpoint: "http://jaeger:4317" # OTLP gRPC endpoint for Jaeger
otlp/traces: # OTLP 导出器,用于跟踪数据
endpoint: "jaeger:4317" # Jaeger 的 OTLP gRPC 端点
tls:
insecure: true # TLS is disabled in the development environment and a certificate needs to be configured in the production environment.
compression: gzip # Enable compression to reduce network bandwidth
retry_on_failure:
enabled: true # Enable retry on failure
initial_interval: 1s # Initial interval for retry
max_interval: 30s # Maximum interval for retry
max_elapsed_time: 300s # Maximum elapsed time for retry
sending_queue:
enabled: true # Enable sending queue
num_consumers: 10 # Number of consumers
queue_size: 5000 # Queue size
otlp/tempo: # OTLP exporter for trace data
endpoint: "http://tempo:4317" # OTLP gRPC endpoint for tempo
insecure: true # 开发环境禁用 TLS,生产环境需配置证书
otlp/tempo: # OTLP 导出器,用于跟踪数据
endpoint: "tempo:4317" # tempo 的 OTLP gRPC 端点
tls:
insecure: true # TLS is disabled in the development environment and a certificate needs to be configured in the production environment.
compression: gzip # Enable compression to reduce network bandwidth
retry_on_failure:
enabled: true # Enable retry on failure
initial_interval: 1s # Initial interval for retry
max_interval: 30s # Maximum interval for retry
max_elapsed_time: 300s # Maximum elapsed time for retry
sending_queue:
enabled: true # Enable sending queue
num_consumers: 10 # Number of consumers
queue_size: 5000 # Queue size
prometheus: # Prometheus exporter for metrics data
endpoint: "0.0.0.0:8889" # Prometheus scraping endpoint
send_timestamps: true # Send timestamp
metric_expiration: 5m # Metric expiration time
resource_to_telemetry_conversion:
enabled: true # Enable resource to telemetry conversion
otlphttp/loki: # Loki exporter for log data
endpoint: "http://loki:3100/otlp"
insecure: true # 开发环境禁用 TLS,生产环境需配置证书
prometheus: # Prometheus 导出器,用于指标数据
endpoint: "0.0.0.0:8889" # Prometheus 刮取端点
namespace: "rustfs" # 指标前缀
send_timestamps: true # 发送时间戳
# enable_open_metrics: true
otlphttp/loki: # Loki 导出器,用于日志数据
# endpoint: "http://loki:3100/otlp/v1/logs"
endpoint: "http://loki:3100/otlp/v1/logs"
tls:
insecure: true
compression: gzip # Enable compression to reduce network bandwidth
extensions:
health_check:
endpoint: 0.0.0.0:13133
pprof:
endpoint: 0.0.0.0:1888
zpages:
endpoint: 0.0.0.0:55679
service:
extensions: [ health_check, pprof, zpages ] # Enable extension
extensions: [ health_check, pprof, zpages ] # 启用扩展
pipelines:
traces:
receivers: [ otlp ]
processors: [ memory_limiter, batch ]
exporters: [ otlp/traces, otlp/tempo ]
processors: [ memory_limiter,batch ]
exporters: [ otlp/traces,otlp/tempo ]
metrics:
receivers: [ otlp ]
processors: [ batch ]
exporters: [ prometheus ]
logs:
receivers: [ otlp ]
processors: [ batch, transform/logs ]
processors: [ batch ]
exporters: [ otlphttp/loki ]
telemetry:
logs:
level: "debug" # Collector log level
encoding: "json" # Log encoding: console or json
level: "info" # Collector 日志级别
metrics:
level: "detailed" # Can be basic, normal, detailed
level: "detailed" # 可以是 basic, normal, detailed
readers:
- periodic:
exporter:
otlp:
protocol: http/protobuf
endpoint: http://otel-collector:4318
- pull:
exporter:
prometheus:
host: '0.0.0.0'
port: 8888
@@ -1 +0,0 @@
*
+7 -44
View File
@@ -13,53 +13,16 @@
# limitations under the License.
global:
scrape_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
evaluation_interval: 15s
external_labels:
cluster: 'rustfs-dev' # Label to identify the cluster
relica: '1' # Replica identifier
scrape_interval: 5s # 刮取间隔
scrape_configs:
- job_name: 'otel-collector-internal'
- job_name: 'otel-collector'
static_configs:
- targets: [ 'otel-collector:8888' ] # Scrape metrics from Collector
scrape_interval: 10s
- job_name: 'rustfs-app-metrics'
- targets: [ 'otel-collector:8888' ] # Collector 刮取指标
- job_name: 'otel-metrics'
static_configs:
- targets: [ 'otel-collector:8889' ] # Application indicators
scrape_interval: 15s
metric_relabel_configs:
- targets: [ 'otel-collector:8889' ] # 应用指标
- job_name: 'tempo'
static_configs:
- targets: [ 'tempo:3200' ] # Scrape metrics from Tempo
- job_name: 'jaeger'
static_configs:
- targets: [ 'jaeger:8888' ] # Jaeger admin port
otlp:
# Recommended attributes to be promoted to labels.
promote_resource_attributes:
- service.instance.id
- service.name
- service.namespace
- cloud.availability_zone
- cloud.region
- container.name
- deployment.environment.name
- k8s.cluster.name
- k8s.container.name
- k8s.cronjob.name
- k8s.daemonset.name
- k8s.deployment.name
- k8s.job.name
- k8s.namespace.name
- k8s.pod.name
- k8s.replicaset.name
- k8s.statefulset.name
# Ingest OTLP data keeping all characters in metric/label names.
translation_strategy: NoUTF8EscapingWithSuffixes
storage:
# OTLP is a push-based protocol, Out of order samples is a common scenario.
tsdb:
out_of_order_time_window: 30m
- targets: [ 'tempo:3200' ]
+1 -3
View File
@@ -18,9 +18,7 @@ distributor:
otlp:
protocols:
grpc:
endpoint: "0.0.0.0:4317"
http:
endpoint: "0.0.0.0:4318"
endpoint: "tempo:4317"
ingester:
max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+1 -1
View File
@@ -67,7 +67,7 @@ To send telemetry data from your application, configure your OpenTelemetry SDK t
For example, in a Rust application using the `rustfs-obs` library:
```bash
export RUSTFS_OBS_ENDPOINT=http://localhost:4318
export RUSTFS_OBS_ENDPOINT=http://localhost:4317
export RUSTFS_OBS_SERVICE_NAME=yourservice
export RUSTFS_OBS_SERVICE_VERSION=1.0.0
export RUSTFS_OBS_ENVIRONMENT=development
-1
View File
@@ -1 +0,0 @@
use flake
+7 -2
View File
@@ -52,19 +52,24 @@ runs:
sudo apt-get install -y \
musl-tools \
build-essential \
lld \
libdbus-1-dev \
libwayland-dev \
libwebkit2gtk-4.1-dev \
libxdo-dev \
pkg-config \
libssl-dev
- name: Install protoc
uses: arduino/setup-protoc@v3
with:
version: "33.1"
version: "31.1"
repo-token: ${{ inputs.github-token }}
- name: Install flatc
uses: Nugine/setup-flatc@v1
with:
version: "25.9.23"
version: "25.2.10"
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
-1
View File
@@ -1 +0,0 @@
../AGENTS.md
+2 -15
View File
@@ -22,21 +22,8 @@ updates:
- package-ecosystem: "cargo" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "weekly"
day: "monday"
timezone: "Asia/Shanghai"
time: "08:00"
ignore:
- dependency-name: "object_store"
versions: [ "0.13.x" ]
interval: "monthly"
groups:
s3s:
update-types:
- "minor"
- "patch"
patterns:
- "s3s"
- "s3s-*"
dependencies:
patterns:
- "*"
- "*"
-103
View File
@@ -1,103 +0,0 @@
# S3 Compatibility Tests Configuration
This directory contains the configuration for running [Ceph S3 compatibility tests](https://github.com/ceph/s3-tests) against RustFS.
## Configuration File
The `s3tests.conf` file is based on the official `s3tests.conf.SAMPLE` from the ceph/s3-tests repository. It uses environment variable substitution via `envsubst` to configure the endpoint and credentials.
### Key Configuration Points
- **Host**: Set via `${S3_HOST}` environment variable (e.g., `rustfs-single` for single-node, `lb` for multi-node)
- **Port**: 9000 (standard RustFS port)
- **Credentials**: Uses `${S3_ACCESS_KEY}` and `${S3_SECRET_KEY}` from workflow environment
- **TLS**: Disabled (`is_secure = False`)
## Test Execution Strategy
### Network Connectivity Fix
Tests run inside a Docker container on the `rustfs-net` network, which allows them to resolve and connect to the RustFS container hostnames. This fixes the "Temporary failure in name resolution" error that occurred when tests ran on the GitHub runner host.
### Performance Optimizations
1. **Parallel Execution**: Uses `pytest-xdist` with `-n 4` to run tests in parallel across 4 workers
2. **Load Distribution**: Uses `--dist=loadgroup` to distribute test groups across workers
3. **Fail-Fast**: Uses `--maxfail=50` to stop after 50 failures, saving time on catastrophic failures
### Feature Filtering
Tests are filtered using pytest markers (`-m`) to skip features not yet supported by RustFS:
- `lifecycle` - Bucket lifecycle policies
- `versioning` - Object versioning
- `s3website` - Static website hosting
- `bucket_logging` - Bucket logging
- `encryption` / `sse_s3` - Server-side encryption
- `cloud_transition` / `cloud_restore` - Cloud storage transitions
- `lifecycle_expiration` / `lifecycle_transition` - Lifecycle operations
This filtering:
1. Reduces test execution time significantly (from 1+ hour to ~10-15 minutes)
2. Focuses on features RustFS currently supports
3. Avoids hundreds of expected failures
## Running Tests Locally
### Single-Node Test
```bash
# Set credentials
export S3_ACCESS_KEY=rustfsadmin
export S3_SECRET_KEY=rustfsadmin
# Start RustFS container
docker run -d --name rustfs-single \
--network rustfs-net \
-e RUSTFS_ADDRESS=0.0.0.0:9000 \
-e RUSTFS_ACCESS_KEY=$S3_ACCESS_KEY \
-e RUSTFS_SECRET_KEY=$S3_SECRET_KEY \
-e RUSTFS_VOLUMES="/data/rustfs0 /data/rustfs1 /data/rustfs2 /data/rustfs3" \
rustfs-ci
# Generate config
export S3_HOST=rustfs-single
envsubst < .github/s3tests/s3tests.conf > /tmp/s3tests.conf
# Run tests
docker run --rm \
--network rustfs-net \
-v /tmp/s3tests.conf:/etc/s3tests.conf:ro \
python:3.12-slim \
bash -c '
apt-get update -qq && apt-get install -y -qq git
git clone --depth 1 https://github.com/ceph/s3-tests.git /s3-tests
cd /s3-tests
pip install -q -r requirements.txt pytest-xdist
S3TEST_CONF=/etc/s3tests.conf pytest -v -n 4 \
s3tests/functional/test_s3.py \
-m "not lifecycle and not versioning and not s3website and not bucket_logging and not encryption and not sse_s3"
'
```
## Test Results Interpretation
- **PASSED**: Test succeeded, feature works correctly
- **FAILED**: Test failed, indicates a potential bug or incompatibility
- **ERROR**: Test setup failed (e.g., network issues, missing dependencies)
- **SKIPPED**: Test skipped due to marker filtering
## Adding New Feature Support
When adding support for a new S3 feature to RustFS:
1. Remove the corresponding marker from the filter in `.github/workflows/e2e-s3tests.yml`
2. Run the tests to verify compatibility
3. Fix any failing tests
4. Update this README to reflect the newly supported feature
## References
- [Ceph S3 Tests Repository](https://github.com/ceph/s3-tests)
- [S3 API Compatibility](https://docs.aws.amazon.com/AmazonS3/latest/API/)
- [pytest-xdist Documentation](https://pytest-xdist.readthedocs.io/)
-193
View File
@@ -1,193 +0,0 @@
# RustFS s3-tests configuration
# Based on: https://github.com/ceph/s3-tests/blob/master/s3tests.conf.SAMPLE
#
# Usage:
# Single-node: S3_HOST=rustfs-single envsubst < s3tests.conf > /tmp/s3tests.conf
# Multi-node: S3_HOST=lb envsubst < s3tests.conf > /tmp/s3tests.conf
[DEFAULT]
## this section is just used for host, port and bucket_prefix
# host set for RustFS - will be substituted via envsubst
host = ${S3_HOST}
# port for RustFS
port = 9000
## say "False" to disable TLS
is_secure = False
## say "False" to disable SSL Verify
ssl_verify = False
[fixtures]
## all the buckets created will start with this prefix;
## {random} will be filled with random characters to pad
## the prefix to 30 characters long, and avoid collisions
bucket prefix = rustfs-{random}-
# all the iam account resources (users, roles, etc) created
# will start with this name prefix
iam name prefix = s3-tests-
# all the iam account resources (users, roles, etc) created
# will start with this path prefix
iam path prefix = /s3-tests/
[s3 main]
# main display_name
display_name = RustFS Tester
# main user_id
user_id = rustfsadmin
# main email
email = tester@rustfs.local
# zonegroup api_name for bucket location
api_name = default
## main AWS access key
access_key = ${S3_ACCESS_KEY}
## main AWS secret key
secret_key = ${S3_SECRET_KEY}
## replace with key id obtained when secret is created, or delete if KMS not tested
#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
## Storage classes
#storage_classes = "LUKEWARM, FROZEN"
## Lifecycle debug interval (default: 10)
#lc_debug_interval = 20
## Restore debug interval (default: 100)
#rgw_restore_debug_interval = 60
#rgw_restore_processor_period = 60
[s3 alt]
# alt display_name
display_name = RustFS Alt Tester
## alt email
email = alt@rustfs.local
# alt user_id
user_id = rustfsalt
# alt AWS access key (must be different from s3 main for many tests)
access_key = ${S3_ALT_ACCESS_KEY}
# alt AWS secret key
secret_key = ${S3_ALT_SECRET_KEY}
#[s3 cloud]
## to run the testcases with "cloud_transition" for transition
## and "cloud_restore" for restore attribute.
## Note: the waiting time may have to tweaked depending on
## the I/O latency to the cloud endpoint.
## host set for cloud endpoint
# host = localhost
## port set for cloud endpoint
# port = 8001
## say "False" to disable TLS
# is_secure = False
## cloud endpoint credentials
# access_key = 0555b35654ad1656d804
# secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
## storage class configured as cloud tier on local rgw server
# cloud_storage_class = CLOUDTIER
## Below are optional -
## Above configured cloud storage class config options
# retain_head_object = false
# allow_read_through = false # change it to enable read_through
# read_through_restore_days = 2
# target_storage_class = Target_SC
# target_path = cloud-bucket
## another regular storage class to test multiple transition rules,
# storage_class = S1
[s3 tenant]
# tenant display_name
display_name = RustFS Tenant Tester
# tenant user_id
# Note: Using same user_id as main to avoid teardown failures.
# RustFS does not currently support multi-tenancy, so the tenant client
# effectively operates as the main user. This ensures nuke_prefixed_buckets()
# in s3-tests teardown can successfully clean up resources.
user_id = rustfsadmin
# tenant AWS access key
access_key = ${S3_ACCESS_KEY}
# tenant AWS secret key
secret_key = ${S3_SECRET_KEY}
# tenant email
email = tenant@rustfs.local
# tenant name
# Note: Empty tenant name to avoid multi-tenant path issues during teardown.
# When s3-tests calls get_tenant_client(), it uses this tenant value in requests.
# An empty value makes the tenant client behave like the main client, preventing
# "bucket not found" errors when teardown tries to clean up test buckets.
tenant =
#following section needs to be added for all sts-tests
[iam]
#used for iam operations in sts-tests
#email
email = s3@rustfs.local
#user_id
user_id = rustfsiam
#access_key
access_key = ${S3_ACCESS_KEY}
#secret_key
secret_key = ${S3_SECRET_KEY}
#display_name
display_name = RustFS IAM User
# iam account root user for iam_account tests
[iam root]
access_key = ${S3_ACCESS_KEY}
secret_key = ${S3_SECRET_KEY}
user_id = RGW11111111111111111
email = account1@rustfs.local
# iam account root user in a different account than [iam root]
[iam alt root]
access_key = ${S3_ACCESS_KEY}
secret_key = ${S3_SECRET_KEY}
user_id = RGW22222222222222222
email = account2@rustfs.local
#following section needs to be added when you want to run Assume Role With Webidentity test
[webidentity]
#used for assume role with web identity test in sts-tests
#all parameters will be obtained from ceph/qa/tasks/keycloak.py
#token=<access_token>
#aud=<obtained after introspecting token>
#sub=<obtained after introspecting token>
#azp=<obtained after introspecting token>
#user_token=<access token for a user, with attribute Department=[Engineering, Marketing>]
#thumbprint=<obtained from x509 certificate>
#KC_REALM=<name of the realm>
+5 -5
View File
@@ -40,11 +40,11 @@ env:
jobs:
security-audit:
name: Security Audit
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Install cargo-audit
uses: taiki-e/install-action@v2
@@ -57,7 +57,7 @@ jobs:
- name: Upload audit results
if: always()
uses: actions/upload-artifact@v6
uses: actions/upload-artifact@v4
with:
name: security-audit-results-${{ github.run_number }}
path: audit-results.json
@@ -65,14 +65,14 @@ jobs:
dependency-review:
name: Dependency Review
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
permissions:
contents: read
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Dependency Review
uses: actions/dependency-review-action@v4
+18 -18
View File
@@ -83,7 +83,7 @@ jobs:
# Build strategy check - determine build type based on trigger
build-check:
name: Build Strategy Check
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
outputs:
should_build: ${{ steps.check.outputs.should_build }}
build_type: ${{ steps.check.outputs.build_type }}
@@ -92,7 +92,7 @@ jobs:
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
with:
fetch-depth: 0
@@ -167,19 +167,19 @@ jobs:
matrix:
include:
# Linux builds
- os: ubicloud-standard-2
- os: ubuntu-latest
target: x86_64-unknown-linux-musl
cross: false
platform: linux
- os: ubicloud-standard-2
- os: ubuntu-latest
target: aarch64-unknown-linux-musl
cross: true
platform: linux
- os: ubicloud-standard-2
- os: ubuntu-latest
target: x86_64-unknown-linux-gnu
cross: false
platform: linux
- os: ubicloud-standard-2
- os: ubuntu-latest
target: aarch64-unknown-linux-gnu
cross: true
platform: linux
@@ -203,7 +203,7 @@ jobs:
# platform: windows
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
with:
fetch-depth: 0
@@ -442,7 +442,7 @@ jobs:
echo "📊 Version: ${VERSION}"
- name: Upload to GitHub artifacts
uses: actions/upload-artifact@v6
uses: actions/upload-artifact@v4
with:
name: ${{ steps.package.outputs.package_name }}
path: "rustfs-*.zip"
@@ -454,7 +454,7 @@ jobs:
OSS_ACCESS_KEY_ID: ${{ secrets.ALICLOUDOSS_KEY_ID }}
OSS_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
OSS_REGION: cn-beijing
OSS_ENDPOINT: https://oss-accelerate.aliyuncs.com
OSS_ENDPOINT: https://oss-cn-beijing.aliyuncs.com
shell: bash
run: |
BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
@@ -532,7 +532,7 @@ jobs:
name: Build Summary
needs: [ build-check, build-rustfs ]
if: always() && needs.build-check.outputs.should_build == 'true'
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
steps:
- name: Build completion summary
shell: bash
@@ -584,7 +584,7 @@ jobs:
name: Create GitHub Release
needs: [ build-check, build-rustfs ]
if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development'
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
permissions:
contents: write
outputs:
@@ -592,7 +592,7 @@ jobs:
release_url: ${{ steps.create.outputs.release_url }}
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
with:
fetch-depth: 0
@@ -670,16 +670,16 @@ jobs:
name: Upload Release Assets
needs: [ build-check, build-rustfs, create-release ]
if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development'
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
permissions:
contents: write
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Download all build artifacts
uses: actions/download-artifact@v7
uses: actions/download-artifact@v5
with:
path: ./artifacts
pattern: rustfs-*
@@ -751,7 +751,7 @@ jobs:
name: Update Latest Version
needs: [ build-check, upload-release-assets ]
if: startsWith(github.ref, 'refs/tags/')
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
steps:
- name: Update latest.json
env:
@@ -801,12 +801,12 @@ jobs:
name: Publish Release
needs: [ build-check, create-release, upload-release-assets ]
if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development'
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Update release notes and publish
env:
+16 -27
View File
@@ -4,7 +4,7 @@
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -62,23 +62,17 @@ on:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
CARGO_BUILD_JOBS: 2
jobs:
skip-check:
name: Skip Duplicate Actions
permissions:
actions: write
contents: read
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
outputs:
should_skip: ${{ steps.skip_check.outputs.should_skip }}
steps:
@@ -89,13 +83,15 @@ jobs:
concurrent_skipping: "same_content_newer"
cancel_others: true
paths_ignore: '["*.md", "docs/**", "deploy/**"]'
# Never skip release events and tag pushes
do_not_skip: '["workflow_dispatch", "schedule", "merge_group", "release", "push"]'
typos:
name: Typos
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@stable
- name: Typos check with custom config file
uses: crate-ci/typos@master
@@ -104,11 +100,13 @@ jobs:
name: Test and Lint
needs: skip-check
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubicloud-standard-4
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- name: Delete huge unnecessary tools folder
run: rm -rf /opt/hostedtoolcache
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -118,9 +116,6 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install cargo-nextest
uses: taiki-e/install-action@nextest
- name: Run tests
run: |
cargo nextest run --all --exclude e2e_test
@@ -136,16 +131,11 @@ jobs:
name: End-to-End Tests
needs: skip-check
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Clean up previous test run
run: |
rm -rf /tmp/rustfs
rm -f /tmp/rustfs.log
uses: actions/checkout@v5
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -159,14 +149,13 @@ jobs:
uses: taiki-e/cache-cargo-install-action@v2
with:
tool: s3s-e2e
git: https://github.com/s3s-project/s3s.git
rev: 4a04a670cf41274d9be9ab65dc36f4aa3f92fbad
git: https://github.com/Nugine/s3s.git
rev: b7714bfaa17ddfa9b23ea01774a1e7bbdbfc2ca3
- name: Build debug binary
run: |
touch rustfs/build.rs
# Limit concurrency to prevent OOM
cargo build -p rustfs --bins --jobs 2
cargo build -p rustfs --bins
- name: Run end-to-end tests
run: |
@@ -175,7 +164,7 @@ jobs:
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v6
uses: actions/upload-artifact@v4
with:
name: e2e-test-logs-${{ github.run_number }}
path: /tmp/rustfs.log
+23 -35
View File
@@ -72,7 +72,7 @@ jobs:
# Check if we should build Docker images
build-check:
name: Docker Build Check
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
outputs:
should_build: ${{ steps.check.outputs.should_build }}
should_push: ${{ steps.check.outputs.should_push }}
@@ -83,7 +83,7 @@ jobs:
create_latest: ${{ steps.check.outputs.create_latest }}
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
with:
fetch-depth: 0
# For workflow_run events, checkout the specific commit that triggered the workflow
@@ -162,11 +162,11 @@ jobs:
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
build_type="prerelease"
is_prerelease=true
# TODO: Temporary change - currently allows alpha versions to also create latest tags
# After the version is stable, you need to remove the following line and restore the original logic (latest is created only for stable versions)
# TODO: 临时修改 - 当前允许 alpha 版本也创建 latest 标签
# 等版本稳定后,需要移除下面这行,恢复原有逻辑(只有稳定版本才创建 latest)
if [[ "$version" == *"alpha"* ]]; then
create_latest=true
echo "🧪 Building Docker image for prerelease: $version (temporarily allowing creation of latest tag)"
echo "🧪 Building Docker image for prerelease: $version (临时允许创建 latest 标签)"
else
echo "🧪 Building Docker image for prerelease: $version"
fi
@@ -215,11 +215,11 @@ jobs:
v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
build_type="prerelease"
is_prerelease=true
# TODO: Temporary change - currently allows alpha versions to also create latest tags
# After the version is stable, you need to remove the if block below and restore the original logic.
# TODO: 临时修改 - 当前允许 alpha 版本也创建 latest 标签
# 等版本稳定后,需要移除下面的 if 块,恢复原有逻辑
if [[ "$input_version" == *"alpha"* ]]; then
create_latest=true
echo "🧪 Building with prerelease version: $input_version (temporarily allowing creation of latest tag)"
echo "🧪 Building with prerelease version: $input_version (临时允许创建 latest 标签)"
else
echo "🧪 Building with prerelease version: $input_version"
fi
@@ -264,22 +264,11 @@ jobs:
name: Build Docker Images
needs: build-check
if: needs.build-check.outputs.should_build == 'true'
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
include:
- variant: musl
file: Dockerfile
suffix: ""
- variant: glibc
file: Dockerfile.glibc
suffix: "-glibc"
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Login to Docker Hub
uses: docker/login-action@v3
@@ -307,7 +296,6 @@ jobs:
VERSION="${{ needs.build-check.outputs.version }}"
SHORT_SHA="${{ needs.build-check.outputs.short_sha }}"
CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}"
VARIANT_SUFFIX="${{ matrix.suffix }}"
# Convert version format for Dockerfile compatibility
case "$VERSION" in
@@ -338,14 +326,14 @@ jobs:
# Generate tags based on build type
# Only support release and prerelease builds (no development builds)
TAGS="${{ env.REGISTRY_DOCKERHUB }}:${VERSION}${VARIANT_SUFFIX}"
TAGS="${{ env.REGISTRY_DOCKERHUB }}:${VERSION}"
# Add channel tags for prereleases and latest for stable
if [[ "$CREATE_LATEST" == "true" ]]; then
# TODO: Temporary change - the current alpha version will also create the latest tag
# After the version is stabilized, the logic here remains unchanged, but the upstream CREATE_LATEST setting needs to be restored.
# Stable release (and temporary alpha versions)
TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX}"
# TODO: 临时修改 - 当前 alpha 版本也会创建 latest 标签
# 等版本稳定后,这里的逻辑保持不变,但上游的 CREATE_LATEST 设置需要恢复
# Stable release (以及临时的 alpha 版本)
TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest"
elif [[ "$BUILD_TYPE" == "prerelease" ]]; then
# Prerelease channel tags (alpha, beta, rc)
if [[ "$VERSION" == *"alpha"* ]]; then
@@ -357,7 +345,7 @@ jobs:
fi
if [[ -n "$CHANNEL" ]]; then
TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}${VARIANT_SUFFIX}"
TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}"
fi
fi
@@ -384,15 +372,15 @@ jobs:
uses: docker/build-push-action@v6
with:
context: .
file: ${{ matrix.file }}
file: Dockerfile
platforms: ${{ env.DOCKER_PLATFORMS }}
push: ${{ needs.build-check.outputs.should_push == 'true' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: |
type=gha,scope=docker-${{ matrix.variant }}
type=gha,scope=docker-binary
cache-to: |
type=gha,mode=max,scope=docker-${{ matrix.variant }}
type=gha,mode=max,scope=docker-binary
build-args: |
BUILDTIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
VERSION=${{ needs.build-check.outputs.version }}
@@ -416,7 +404,7 @@ jobs:
name: Docker Build Summary
needs: [ build-check, build-docker ]
if: always() && needs.build-check.outputs.should_build == 'true'
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
steps:
- name: Docker build completion summary
run: |
@@ -441,10 +429,10 @@ jobs:
"prerelease")
echo "🧪 Prerelease Docker image has been built with ${VERSION} tags"
echo "⚠️ This is a prerelease image - use with caution"
# TODO: Temporary change - alpha versions currently create the latest tag
# After the version is stable, you need to restore the following prompt information
# TODO: 临时修改 - alpha 版本当前会创建 latest 标签
# 等版本稳定后,需要恢复下面的提示信息
if [[ "$VERSION" == *"alpha"* ]] && [[ "$CREATE_LATEST" == "true" ]]; then
echo "🏷️ Latest tag has been created for alpha version (temporary measures)"
echo "🏷️ Latest tag has been created for alpha version (临时措施)"
else
echo "🚫 Latest tag NOT created for prerelease"
fi
-422
View File
@@ -1,422 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: e2e-s3tests
on:
workflow_dispatch:
inputs:
test-mode:
description: "Test mode to run"
required: true
type: choice
default: "single"
options:
- single
- multi
xdist:
description: "Enable pytest-xdist (parallel). '0' to disable."
required: false
default: "0"
maxfail:
description: "Stop after N failures (debug friendly)"
required: false
default: "1"
markexpr:
description: "pytest -m expression (feature filters)"
required: false
default: "not lifecycle and not versioning and not s3website and not bucket_logging and not encryption"
env:
# main user
S3_ACCESS_KEY: rustfsadmin
S3_SECRET_KEY: rustfsadmin
# alt user (must be different from main for many s3-tests)
S3_ALT_ACCESS_KEY: rustfsalt
S3_ALT_SECRET_KEY: rustfsalt
S3_REGION: us-east-1
RUST_LOG: info
PLATFORM: linux/amd64
defaults:
run:
shell: bash
jobs:
s3tests-single:
if: github.event.inputs.test-mode == 'single'
runs-on: ubicloud-standard-2
timeout-minutes: 120
steps:
- uses: actions/checkout@v6
- name: Enable buildx
uses: docker/setup-buildx-action@v3
- name: Build RustFS image (source, cached)
run: |
DOCKER_BUILDKIT=1 docker buildx build --load \
--platform ${PLATFORM} \
--cache-from type=gha \
--cache-to type=gha,mode=max \
-t rustfs-ci \
-f Dockerfile.source .
- name: Create network
run: docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
- name: Remove existing rustfs-single (if any)
run: docker rm -f rustfs-single >/dev/null 2>&1 || true
- name: Start single RustFS
run: |
docker run -d --name rustfs-single \
--network rustfs-net \
-p 9000:9000 \
-e RUSTFS_ADDRESS=0.0.0.0:9000 \
-e RUSTFS_ACCESS_KEY=$S3_ACCESS_KEY \
-e RUSTFS_SECRET_KEY=$S3_SECRET_KEY \
-e RUSTFS_VOLUMES="/data/rustfs0 /data/rustfs1 /data/rustfs2 /data/rustfs3" \
-v /tmp/rustfs-single:/data \
rustfs-ci
- name: Wait for RustFS ready
run: |
for i in {1..60}; do
if curl -sf http://127.0.0.1:9000/health >/dev/null 2>&1; then
echo "RustFS is ready"
exit 0
fi
if [ "$(docker inspect -f '{{.State.Running}}' rustfs-single 2>/dev/null)" != "true" ]; then
echo "RustFS container not running" >&2
docker logs rustfs-single || true
exit 1
fi
sleep 2
done
echo "Health check timed out" >&2
docker logs rustfs-single || true
exit 1
- name: Generate s3tests config
run: |
export S3_HOST=127.0.0.1
envsubst < .github/s3tests/s3tests.conf > s3tests.conf
- name: Provision s3-tests alt user (required by suite)
run: |
python3 -m pip install --user --upgrade pip awscurl
export PATH="$HOME/.local/bin:$PATH"
# Admin API requires AWS SigV4 signing. awscurl is used by RustFS codebase as well.
awscurl \
--service s3 \
--region "${S3_REGION}" \
--access_key "${S3_ACCESS_KEY}" \
--secret_key "${S3_SECRET_KEY}" \
-X PUT \
-H 'Content-Type: application/json' \
-d '{"secretKey":"'"${S3_ALT_SECRET_KEY}"'","status":"enabled","policy":"readwrite"}' \
"http://127.0.0.1:9000/rustfs/admin/v3/add-user?accessKey=${S3_ALT_ACCESS_KEY}"
# Explicitly attach built-in policy via policy mapping.
# s3-tests relies on alt client being able to ListBuckets during setup cleanup.
awscurl \
--service s3 \
--region "${S3_REGION}" \
--access_key "${S3_ACCESS_KEY}" \
--secret_key "${S3_SECRET_KEY}" \
-X PUT \
"http://127.0.0.1:9000/rustfs/admin/v3/set-user-or-group-policy?policyName=readwrite&userOrGroup=${S3_ALT_ACCESS_KEY}&isGroup=false"
# Sanity check: alt user can list buckets (should not be AccessDenied).
awscurl \
--service s3 \
--region "${S3_REGION}" \
--access_key "${S3_ALT_ACCESS_KEY}" \
--secret_key "${S3_ALT_SECRET_KEY}" \
-X GET \
"http://127.0.0.1:9000/" >/dev/null
- name: Prepare s3-tests
run: |
python3 -m pip install --user --upgrade pip tox
export PATH="$HOME/.local/bin:$PATH"
git clone --depth 1 https://github.com/ceph/s3-tests.git s3-tests
- name: Run ceph s3-tests (debug friendly)
run: |
export PATH="$HOME/.local/bin:$PATH"
mkdir -p artifacts/s3tests-single
cd s3-tests
set -o pipefail
MAXFAIL="${{ github.event.inputs.maxfail }}"
if [ -z "$MAXFAIL" ]; then MAXFAIL="1"; fi
MARKEXPR="${{ github.event.inputs.markexpr }}"
if [ -z "$MARKEXPR" ]; then MARKEXPR="not lifecycle and not versioning and not s3website and not bucket_logging and not encryption"; fi
XDIST="${{ github.event.inputs.xdist }}"
if [ -z "$XDIST" ]; then XDIST="0"; fi
XDIST_ARGS=""
if [ "$XDIST" != "0" ]; then
# Add pytest-xdist to requirements.txt so tox installs it inside
# its virtualenv. Installing outside tox does NOT work.
echo "pytest-xdist" >> requirements.txt
XDIST_ARGS="-n $XDIST --dist=loadgroup"
fi
# Run tests from s3tests/functional (boto2+boto3 combined directory).
S3TEST_CONF=${GITHUB_WORKSPACE}/s3tests.conf \
tox -- \
-vv -ra --showlocals --tb=long \
--maxfail="$MAXFAIL" \
--junitxml=${GITHUB_WORKSPACE}/artifacts/s3tests-single/junit.xml \
$XDIST_ARGS \
s3tests/functional/test_s3.py \
-m "$MARKEXPR" \
2>&1 | tee ${GITHUB_WORKSPACE}/artifacts/s3tests-single/pytest.log
- name: Collect RustFS logs
if: always()
run: |
mkdir -p artifacts/rustfs-single
docker logs rustfs-single > artifacts/rustfs-single/rustfs.log 2>&1 || true
docker inspect rustfs-single > artifacts/rustfs-single/inspect.json || true
- name: Upload artifacts
if: always() && env.ACT != 'true'
uses: actions/upload-artifact@v6
with:
name: s3tests-single
path: artifacts/**
s3tests-multi:
if: github.event_name == 'workflow_dispatch' && github.event.inputs.test-mode == 'multi'
runs-on: ubicloud-standard-2
timeout-minutes: 150
steps:
- uses: actions/checkout@v6
- name: Enable buildx
uses: docker/setup-buildx-action@v3
- name: Build RustFS image (source, cached)
run: |
DOCKER_BUILDKIT=1 docker buildx build --load \
--platform ${PLATFORM} \
--cache-from type=gha \
--cache-to type=gha,mode=max \
-t rustfs-ci \
-f Dockerfile.source .
- name: Prepare cluster compose
run: |
cat > compose.yml <<'EOF'
services:
rustfs1:
image: rustfs-ci
hostname: rustfs1
networks: [rustfs-net]
environment:
RUSTFS_ADDRESS: "0.0.0.0:9000"
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
RUSTFS_VOLUMES: "/data/rustfs0 /data/rustfs1 /data/rustfs2 /data/rustfs3"
volumes:
- rustfs1-data:/data
rustfs2:
image: rustfs-ci
hostname: rustfs2
networks: [rustfs-net]
environment:
RUSTFS_ADDRESS: "0.0.0.0:9000"
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
RUSTFS_VOLUMES: "/data/rustfs0 /data/rustfs1 /data/rustfs2 /data/rustfs3"
volumes:
- rustfs2-data:/data
rustfs3:
image: rustfs-ci
hostname: rustfs3
networks: [rustfs-net]
environment:
RUSTFS_ADDRESS: "0.0.0.0:9000"
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
RUSTFS_VOLUMES: "/data/rustfs0 /data/rustfs1 /data/rustfs2 /data/rustfs3"
volumes:
- rustfs3-data:/data
rustfs4:
image: rustfs-ci
hostname: rustfs4
networks: [rustfs-net]
environment:
RUSTFS_ADDRESS: "0.0.0.0:9000"
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
RUSTFS_VOLUMES: "/data/rustfs0 /data/rustfs1 /data/rustfs2 /data/rustfs3"
volumes:
- rustfs4-data:/data
lb:
image: haproxy:2.9
hostname: lb
networks: [rustfs-net]
ports:
- "9000:9000"
volumes:
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
networks:
rustfs-net:
name: rustfs-net
volumes:
rustfs1-data:
rustfs2-data:
rustfs3-data:
rustfs4-data:
EOF
cat > haproxy.cfg <<'EOF'
defaults
mode http
timeout connect 5s
timeout client 30s
timeout server 30s
frontend fe_s3
bind *:9000
default_backend be_s3
backend be_s3
balance roundrobin
server s1 rustfs1:9000 check
server s2 rustfs2:9000 check
server s3 rustfs3:9000 check
server s4 rustfs4:9000 check
EOF
- name: Launch cluster
run: docker compose -f compose.yml up -d
- name: Wait for LB ready
run: |
for i in {1..90}; do
if curl -sf http://127.0.0.1:9000/health >/dev/null 2>&1; then
echo "Load balancer is ready"
exit 0
fi
sleep 2
done
echo "LB or backend not ready" >&2
docker compose -f compose.yml logs --tail=200 || true
exit 1
- name: Generate s3tests config
run: |
export S3_HOST=127.0.0.1
envsubst < .github/s3tests/s3tests.conf > s3tests.conf
- name: Provision s3-tests alt user (required by suite)
run: |
python3 -m pip install --user --upgrade pip awscurl
export PATH="$HOME/.local/bin:$PATH"
awscurl \
--service s3 \
--region "${S3_REGION}" \
--access_key "${S3_ACCESS_KEY}" \
--secret_key "${S3_SECRET_KEY}" \
-X PUT \
-H 'Content-Type: application/json' \
-d '{"secretKey":"'"${S3_ALT_SECRET_KEY}"'","status":"enabled","policy":"readwrite"}' \
"http://127.0.0.1:9000/rustfs/admin/v3/add-user?accessKey=${S3_ALT_ACCESS_KEY}"
awscurl \
--service s3 \
--region "${S3_REGION}" \
--access_key "${S3_ACCESS_KEY}" \
--secret_key "${S3_SECRET_KEY}" \
-X PUT \
"http://127.0.0.1:9000/rustfs/admin/v3/set-user-or-group-policy?policyName=readwrite&userOrGroup=${S3_ALT_ACCESS_KEY}&isGroup=false"
awscurl \
--service s3 \
--region "${S3_REGION}" \
--access_key "${S3_ALT_ACCESS_KEY}" \
--secret_key "${S3_ALT_SECRET_KEY}" \
-X GET \
"http://127.0.0.1:9000/" >/dev/null
- name: Prepare s3-tests
run: |
python3 -m pip install --user --upgrade pip tox
export PATH="$HOME/.local/bin:$PATH"
git clone --depth 1 https://github.com/ceph/s3-tests.git s3-tests
- name: Run ceph s3-tests (multi, debug friendly)
run: |
export PATH="$HOME/.local/bin:$PATH"
mkdir -p artifacts/s3tests-multi
cd s3-tests
set -o pipefail
MAXFAIL="${{ github.event.inputs.maxfail }}"
if [ -z "$MAXFAIL" ]; then MAXFAIL="1"; fi
MARKEXPR="${{ github.event.inputs.markexpr }}"
if [ -z "$MARKEXPR" ]; then MARKEXPR="not lifecycle and not versioning and not s3website and not bucket_logging and not encryption"; fi
XDIST="${{ github.event.inputs.xdist }}"
if [ -z "$XDIST" ]; then XDIST="0"; fi
XDIST_ARGS=""
if [ "$XDIST" != "0" ]; then
# Add pytest-xdist to requirements.txt so tox installs it inside
# its virtualenv. Installing outside tox does NOT work.
echo "pytest-xdist" >> requirements.txt
XDIST_ARGS="-n $XDIST --dist=loadgroup"
fi
# Run tests from s3tests/functional (boto2+boto3 combined directory).
S3TEST_CONF=${GITHUB_WORKSPACE}/s3tests.conf \
tox -- \
-vv -ra --showlocals --tb=long \
--maxfail="$MAXFAIL" \
--junitxml=${GITHUB_WORKSPACE}/artifacts/s3tests-multi/junit.xml \
$XDIST_ARGS \
s3tests/functional/test_s3.py \
-m "$MARKEXPR" \
2>&1 | tee ${GITHUB_WORKSPACE}/artifacts/s3tests-multi/pytest.log
- name: Collect logs
if: always()
run: |
mkdir -p artifacts/cluster
docker compose -f compose.yml logs --no-color > artifacts/cluster/cluster.log 2>&1 || true
- name: Upload artifacts
if: always() && env.ACT != 'true'
uses: actions/upload-artifact@v6
with:
name: s3tests-multi
path: artifacts/**
-94
View File
@@ -1,94 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Publish helm chart to artifacthub
on:
workflow_run:
workflows: [ "Build and Release" ]
types: [ completed ]
permissions:
contents: read
env:
new_version: ${{ github.event.workflow_run.head_branch }}
jobs:
build-helm-package:
runs-on: ubicloud-standard-2
# Only run on successful builds triggered by tag pushes (version format: x.y.z or x.y.z-suffix)
if: |
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.event == 'push' &&
contains(github.event.workflow_run.head_branch, '.')
steps:
- name: Checkout helm chart repo
uses: actions/checkout@v6
- name: Replace chart app version
run: |
set -e
set -x
old_version=$(grep "^appVersion:" helm/rustfs/Chart.yaml | awk '{print $2}')
sed -i "s/$old_version/$new_version/g" helm/rustfs/Chart.yaml
- name: Set up Helm
uses: azure/setup-helm@v4.3.0
- name: Package Helm Chart
run: |
cp helm/README.md helm/rustfs/
package_version=$(echo $new_version | awk -F '-' '{print $2}' | awk -F '.' '{print $NF}')
helm package ./helm/rustfs --destination helm/rustfs/ --version "0.0.$package_version"
- name: Upload helm package as artifact
uses: actions/upload-artifact@v6
with:
name: helm-package
path: helm/rustfs/*.tgz
retention-days: 1
publish-helm-package:
runs-on: ubicloud-standard-2
needs: [ build-helm-package ]
steps:
- name: Checkout helm package repo
uses: actions/checkout@v6
with:
repository: rustfs/helm
token: ${{ secrets.RUSTFS_HELM_PACKAGE }}
- name: Download helm package
uses: actions/download-artifact@v7
with:
name: helm-package
path: ./
- name: Set up helm
uses: azure/setup-helm@v4.3.0
- name: Generate index
run: helm repo index . --url https://charts.rustfs.com
- name: Push helm package and index file
run: |
git config --global user.name "${{ secrets.USERNAME }}"
git config --global user.email "${{ secrets.EMAIL_ADDRESS }}"
git status .
git add .
git commit -m "Update rustfs helm package with $new_version."
git push origin main
+1 -1
View File
@@ -25,7 +25,7 @@ permissions:
jobs:
build:
runs-on: ubicloud-standard-4
runs-on: ubuntu-latest
steps:
- uses: usthe/issues-translate-action@v2.7
with:
+6 -6
View File
@@ -40,11 +40,11 @@ env:
jobs:
performance-profile:
name: Performance Profiling
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -107,7 +107,7 @@ jobs:
- name: Upload profile data
if: steps.profiling.outputs.profile_generated == 'true'
uses: actions/upload-artifact@v6
uses: actions/upload-artifact@v4
with:
name: performance-profile-${{ github.run_number }}
path: samply-profile.json
@@ -115,11 +115,11 @@ jobs:
benchmark:
name: Benchmark Tests
runs-on: ubicloud-standard-2
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -135,7 +135,7 @@ jobs:
tee benchmark-results.json
- name: Upload benchmark results
uses: actions/upload-artifact@v6
uses: actions/upload-artifact@v4
with:
name: benchmark-results-${{ github.run_number }}
path: benchmark-results.json
-20
View File
@@ -1,20 +0,0 @@
name: "Mark stale issues"
on:
schedule:
- cron: "30 1 * * *"
jobs:
stale:
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v9
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
stale-issue-message: 'This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs.'
stale-issue-label: 'stale'
## Mark if there is no activity for more than 7 days
days-before-stale: 7
# If no one responds after 3 days, the tag will be closed.
days-before-close: 3
# These tags are exempt and will not close automatically.
exempt-issue-labels: 'pinned,security'
+1 -17
View File
@@ -2,12 +2,9 @@
.DS_Store
.idea
.vscode
.direnv/
/test
/logs
/data
/docs
/rustfs-data/
.devcontainer
rustfs/static/*
!rustfs/static/.gitkeep
@@ -26,17 +23,4 @@ profile.json
*.go
*.pb
*.svg
deploy/logs/*.log.*
artifacts/
# s3-tests local artifacts (root directory only)
/s3-tests/
/s3-tests-local/
/s3tests.conf
/s3tests.conf.*
*.events
*.audit
*.snappy
PR_DESCRIPTION.md
IMPLEMENTATION_PLAN.md
scripts/s3-tests/selected_tests.txt
docs
deploy/logs/*.log.*
-32
View File
@@ -1,32 +0,0 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: local
hooks:
- id: cargo-fmt
name: cargo fmt
entry: cargo fmt --all --check
language: system
types: [rust]
pass_filenames: false
- id: cargo-clippy
name: cargo clippy
entry: cargo clippy --all-targets --all-features -- -D warnings
language: system
types: [rust]
pass_filenames: false
- id: cargo-check
name: cargo check
entry: cargo check --all-targets
language: system
types: [rust]
pass_filenames: false
- id: cargo-test
name: cargo test
entry: bash -c 'cargo test --workspace --exclude e2e_test && cargo test --all --doc'
language: system
types: [rust]
pass_filenames: false
+11 -148
View File
@@ -1,31 +1,9 @@
{
// 使用 IntelliSense 了解相关属性。
// 悬停以查看现有属性的描述。
// 欲了解更多信息,请访问: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "lldb",
"request": "launch",
"name": "Debug(only) executable 'rustfs'",
"env": {
"RUST_LOG": "rustfs=info,ecstore=info,s3s=info,iam=info",
"RUSTFS_SKIP_BACKGROUND_TASK": "on"
//"RUSTFS_OBS_LOG_DIRECTORY": "./deploy/logs",
// "RUSTFS_POLICY_PLUGIN_URL":"http://localhost:8181/v1/data/rustfs/authz/allow",
// "RUSTFS_POLICY_PLUGIN_AUTH_TOKEN":"your-opa-token"
},
"program": "${workspaceFolder}/target/debug/rustfs",
"args": [
"--access-key",
"rustfsadmin",
"--secret-key",
"rustfsadmin",
"--address",
"0.0.0.0:9010",
"--server-domains",
"127.0.0.1:9010",
"./target/volume/test{1...4}"
],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
@@ -44,7 +22,6 @@
"env": {
"RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=debug",
"RUSTFS_SKIP_BACKGROUND_TASK": "on",
//"RUSTFS_OBS_LOG_DIRECTORY": "./deploy/logs",
// "RUSTFS_POLICY_PLUGIN_URL":"http://localhost:8181/v1/data/rustfs/authz/allow",
// "RUSTFS_POLICY_PLUGIN_AUTH_TOKEN":"your-opa-token"
},
@@ -89,8 +66,12 @@
"test",
"--no-run",
"--lib",
"--package=rustfs-ecstore"
]
"--package=ecstore"
],
"filter": {
"name": "ecstore",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
@@ -99,58 +80,15 @@
"name": "Debug executable target/debug/rustfs",
"type": "lldb",
"request": "launch",
"cargo": {
"args": [
"run",
"--bin",
"rustfs",
"-j",
"1",
"--profile",
"dev"
]
},
"program": "${workspaceFolder}/target/debug/rustfs",
"args": [],
"cwd": "${workspaceFolder}",
//"stopAtEntry": false,
//"preLaunchTask": "cargo build",
"env": {
"RUSTFS_ACCESS_KEY": "rustfsadmin",
"RUSTFS_SECRET_KEY": "rustfsadmin",
//"RUSTFS_VOLUMES": "./target/volume/test{1...4}",
"RUSTFS_ADDRESS": ":9000",
"RUSTFS_CONSOLE_ENABLE": "true",
// "RUSTFS_OBS_TRACE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", // jeager otlp http endpoint
// "RUSTFS_OBS_METRIC_ENDPOINT": "http://127.0.0.1:4318/v1/metrics", // default otlp http endpoint
// "RUSTFS_OBS_LOG_ENDPOINT": "http://127.0.0.1:4318/v1/logs", // default otlp http endpoint
// "RUSTFS_COMPRESS_ENABLE": "true",
"RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
"RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
"RUST_LOG":"rustfs=debug,ecstore=debug,s3s=debug,iam=debug",
},
"sourceLanguages": [
"rust"
],
},
{
"type": "lldb",
"request": "launch",
"name": "Debug test_lifecycle_transition_basic",
"cargo": {
"args": [
"test",
"-p",
"rustfs-scanner",
"--test",
"lifecycle_integration_test",
"serial_tests::test_lifecycle_transition_basic",
"-j",
"1"
]
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"name": "Debug executable target/debug/test",
"type": "lldb",
@@ -170,81 +108,6 @@
"sourceLanguages": [
"rust"
],
},
{
"name": "Debug executable target/debug/rustfs with sse",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/target/debug/rustfs",
"args": [],
"cwd": "${workspaceFolder}",
//"stopAtEntry": false,
//"preLaunchTask": "cargo build",
"env": {
"RUSTFS_ACCESS_KEY": "rustfsadmin",
"RUSTFS_SECRET_KEY": "rustfsadmin",
"RUSTFS_VOLUMES": "./target/volumes/test{1...4}",
"RUSTFS_ADDRESS": ":9000",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
"RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
// "RUSTFS_OBS_TRACE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", // jeager otlp http endpoint
// "RUSTFS_OBS_METRIC_ENDPOINT": "http://127.0.0.1:4318/v1/metrics", // default otlp http endpoint
// "RUSTFS_OBS_LOG_ENDPOINT": "http://127.0.0.1:4318/v1/logs", // default otlp http endpoint
// "RUSTFS_COMPRESS_ENABLE": "true",
// 1. simple sse test key (no kms system)
// "__RUSTFS_SSE_SIMPLE_CMK": "2dfNXGHlsEflGVCxb+5DIdGEl1sIvtwX+QfmYasi5QM=",
// 2. kms local backend test key
"RUSTFS_KMS_ENABLE": "true",
"RUSTFS_KMS_BACKEND": "local",
"RUSTFS_KMS_KEY_DIR": "./target/kms-key-dir",
"RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
"RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
// 3. kms vault backend test key
// "RUSTFS_KMS_ENABLE": "true",
// "RUSTFS_KMS_BACKEND": "vault",
// "RUSTFS_KMS_VAULT_ADDRESS": "http://127.0.0.1:8200",
// "RUSTFS_KMS_VAULT_TOKEN": "Dev Token",
// "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
},
"sourceLanguages": [
"rust"
],
},
{
"name": "E2E test executable target/debug/rustfs",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/target/debug/rustfs",
"args": [],
"cwd": "${workspaceFolder}",
//"stopAtEntry": false,
//"preLaunchTask": "cargo build",
"env": {
"RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=debug",
"RUST_BACKTRACE": "full",
"RUSTFS_ACCESS_KEY": "rustfsadmin",
"RUSTFS_SECRET_KEY": "rustfsadmin",
"RUSTFS_VOLUMES": "./target/e2e-test/test{1...4}",
"RUSTFS_REGION": "us-east-1",
"RUSTFS_ADDRESS": ":9000",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
"RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
"RUSTFS_KMS_ENABLE": "true",
"RUSTFS_KMS_BACKEND": "local",
"RUSTFS_KMS_KEY_DIR": "./target/e2e-key-dir",
"RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
"RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
},
"sourceLanguages": [
"rust"
],
},
}
]
}
+9 -86
View File
@@ -1,101 +1,24 @@
# Repository Guidelines
This file provides guidance for AI agents and developers working with code in this repository.
## Project Overview
RustFS is a high-performance distributed object storage software built with Rust, providing S3-compatible APIs and advanced features like data lakes, AI, and big data support. It's designed as an alternative to MinIO with better performance and a more business-friendly Apache 2.0 license.
## ⚠️ Pre-Commit Checklist (MANDATORY)
**Before EVERY commit, you MUST run and pass ALL of the following:**
```bash
cargo fmt --all --check # Code formatting
cargo clippy --all-targets --all-features -- -D warnings # Lints
cargo test --workspace --exclude e2e_test # Unit tests
```
Or simply run `make pre-commit` which covers all checks. **DO NOT commit if any check fails.**
## Communication Rules
- Respond to the user in Chinese; use English in all other contexts.
- Code and documentation must be written in English only. Chinese text is allowed solely as test data/fixtures when a case explicitly requires Chinese-language content for validation.
- **Pull Request titles and descriptions must be written in English** to ensure consistency and accessibility for all contributors.
## Project Structure
The workspace root hosts shared dependencies in `Cargo.toml`. The service binary lives under `rustfs/src/main.rs`, while reusable crates sit in `crates/` (including `ecstore`, `iam`, `kms`, `madmin`, `s3select-api`, `s3select-query`, `config`, `crypto`, `lock`, `filemeta`, `rio`, `common`, `protos`, `audit-logger`, `notify`, `obs`, `workers`, `appauth`, `ahm`, `mcp`, `signer`, `checksums`, `utils`, `zip`, `targets`, and `e2e_test`). Deployment manifests are under `deploy/`, Docker assets sit at the root, and automation lives in `scripts/`.
### Core Architecture
- **Main Binary (`rustfs/`):** Entry point at `rustfs/src/main.rs`, includes admin, auth, config, server, storage, license management, and profiling modules.
- **Key Crates:** Cargo workspace with 25+ crates supporting S3-compatible APIs, erasure coding storage, IAM, KMS, S3 Select, and observability.
- **Build System:** Custom `build-rustfs.sh` script, multi-architecture Docker builds, Make/Just task runners, cross-compilation support.
## Project Structure & Module Organization
The workspace root hosts shared dependencies in `Cargo.toml`. The service binary lives under `rustfs/src/main.rs`, while reusable crates sit in `crates/` (`crypto`, `iam`, `kms`, and `e2e_test`). Local fixtures for standalone flows reside in `test_standalone/`, deployment manifests are under `deploy/`, Docker assets sit at the root, and automation lives in `scripts/`. Skim each crates README or module docs before contributing changes.
## Build, Test, and Development Commands
Run `cargo check --all-targets` for fast validation. Build release binaries via `cargo build --release` or the pipeline-aligned `make build`. Use `./build-rustfs.sh --dev` for iterative development and `./build-rustfs.sh --platform <target>` for cross-compiles. Prefer `make pre-commit` before pushing to cover formatting, clippy, checks, and tests.
Always ensure `cargo fmt --all --check`, `cargo test --workspace --exclude e2e_test`, and `cargo clippy --all-targets --all-features -- -D warnings` complete successfully after each code change to keep the tree healthy and warning-free.
### Quick Commands
- `cargo check --all-targets` - Fast validation
- `cargo build --release` or `make build` - Release build
- `./build-rustfs.sh --dev` - Development build with debug symbols
- `make pre-commit` - Run all quality checks (fmt, clippy, check, test)
### Testing
- `cargo test --workspace --exclude e2e_test` - Unit tests
- `cargo test --package e2e_test` - E2E tests
- For KMS tests: `NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1`
### Cross-Platform Builds
- `./build-rustfs.sh --platform x86_64-unknown-linux-musl` - Build for musl
- `./build-rustfs.sh --platform aarch64-unknown-linux-gnu` - Build for ARM64
- `make build-cross-all` - Build all supported architectures
## Coding Style & Safety Requirements
- **Formatting:** Follow `rustfmt.toml` (130-column width). Use `snake_case` for items, `PascalCase` for types, `SCREAMING_SNAKE_CASE` for constants.
- **Safety:** `unsafe_code = "deny"` enforced at workspace level. Never use `unwrap()`, `expect()`, or panic-inducing code except in tests.
- **Error Handling:** Prefer `anyhow` for applications, `thiserror` for libraries. Use proper error handling with `Result<T, E>` and `Option<T>`.
- **Async:** Keep async code non-blocking. Offload CPU-heavy work with `tokio::task::spawn_blocking` when necessary.
- **Language:** Code comments, function names, variable names, and all text in source files must be in English only.
## Coding Style & Naming Conventions
Formatting follows the repo `rustfmt.toml` (130-column width). Use `snake_case` for items, `PascalCase` for types, and `SCREAMING_SNAKE_CASE` for constants. Avoid `unwrap()` or `expect()` outside tests; bubble errors with `Result` and crate-specific `thiserror` types. Keep async code non-blocking and offload CPU-heavy work with `tokio::task::spawn_blocking` when necessary.
## Testing Guidelines
Co-locate unit tests with their modules and give behavior-led names. Integration suites belong in each crate's `tests/` directory, while exhaustive end-to-end scenarios live in `crates/e2e_test/`. When fixing bugs or adding features, include regression tests that capture the new behavior.
## KMS (Key Management Service)
- **Implementation:** Complete with Local and Vault backends, auto-configures on startup with `--kms-enable` flag.
- **Encryption:** Full S3-compatible server-side encryption (SSE-S3, SSE-KMS, SSE-C).
- **Testing:** Comprehensive E2E tests in `crates/e2e_test/src/kms/`. Requires proxy bypass for local testing.
- **Key Files:** `crates/kms/`, `rustfs/src/storage/ecfs.rs`, `rustfs/src/admin/handlers/kms*.rs`
## Environment Variables
- `RUSTFS_ENABLE_SCANNER` - Enable/disable background data scanner (default: true)
- `RUSTFS_ENABLE_HEAL` - Enable/disable auto-heal functionality (default: true)
- For KMS tests: `NO_PROXY=127.0.0.1,localhost` and clear proxy environment variables
Co-locate unit tests with their modules and give behavior-led names such as `handles_expired_token`. Integration suites belong in each crates `tests/` directory, while exhaustive end-to-end scenarios live in `crates/e2e_test/`. Run `cargo test --workspace --exclude e2e_test` during iteration, `cargo nextest run --all --exclude e2e_test` when available, and finish with `cargo test --all` before requesting review. Use `NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY=` for KMS e2e tests.
When fixing bugs or adding features, include regression tests that capture the new behavior so future changes cannot silently break it.
## Commit & Pull Request Guidelines
Work on feature branches (e.g., `feat/...`) after syncing `main`. Follow Conventional Commits under 72 characters. Each commit must compile, format cleanly, and pass `make pre-commit`.
**Pull Request Requirements:**
- PR titles and descriptions **MUST be written in English**
- Open PRs with a concise summary, note verification commands, link relevant issues
- Follow the PR template format and fill in all required sections
- Wait for reviewer approval before merging
Work on feature branches (e.g., `feat/...`) after syncing `main`. Follow Conventional Commits under 72 characters (e.g., `feat: add kms key rotation`). Each commit must compile, format cleanly, and pass `make pre-commit`. Open PRs with a concise summary, note verification commands, link relevant issues, and wait for reviewer approval.
## Security & Configuration Tips
Do not commit secrets or cloud credentials; prefer environment variables or vault tooling. Review IAM- and KMS-related changes with a second maintainer. Confirm proxy settings before running sensitive tests to avoid leaking traffic outside localhost.
## Important Reminders
- Always compile after code changes: Use `cargo build` to catch errors early
- Don't bypass tests: All functionality must be properly tested, not worked around
- Use proper error handling: Never use `unwrap()` or `expect()` in production code (except tests)
- Do what has been asked; nothing more, nothing less
- NEVER create files unless they're absolutely necessary for achieving your goal
- ALWAYS prefer editing an existing file to creating a new one
- NEVER proactively create documentation files (*.md) or README files unless explicitly requested
- NEVER commit PR description files (e.g., PR_DESCRIPTION.md): These are temporary reference files for creating pull requests and should remain local only
+3
View File
@@ -83,3 +83,6 @@ that body of laws known as conflict of laws. The parties expressly agree that th
for the International Sale of Goods will not apply. Any legal action or proceeding arising under this Agreement will be
brought exclusively in the courts located in Beijing, China, and the parties hereby irrevocably consent to the personal
jurisdiction and venue therein.
For your reading convenience, this Agreement is written in parallel English and Chinese sections. To the extent there is
a conflict between the English and Chinese sections, the English sections shall govern.
+275
View File
@@ -0,0 +1,275 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview
RustFS is a high-performance distributed object storage software built with Rust, providing S3-compatible APIs and
advanced features like data lakes, AI, and big data support. It's designed as an alternative to MinIO with better
performance and a more business-friendly Apache 2.0 license.
## Build Commands
### Primary Build Commands
- `cargo build --release` - Build the main RustFS binary
- `./build-rustfs.sh` - Recommended build script that handles console resources and cross-platform compilation
- `./build-rustfs.sh --dev` - Development build with debug symbols
- `make build` or `just build` - Use Make/Just for standardized builds
### Platform-Specific Builds
- `./build-rustfs.sh --platform x86_64-unknown-linux-musl` - Build for musl target
- `./build-rustfs.sh --platform aarch64-unknown-linux-gnu` - Build for ARM64
- `make build-musl` or `just build-musl` - Build musl variant
- `make build-cross-all` - Build all supported architectures
### Testing Commands
- `cargo test --workspace --exclude e2e_test` - Run unit tests (excluding e2e tests)
- `cargo nextest run --all --exclude e2e_test` - Use nextest if available (faster)
- `cargo test --all --doc` - Run documentation tests
- `make test` or `just test` - Run full test suite
- `make pre-commit` - Run all quality checks (fmt, clippy, check, test)
### End-to-End Testing
- `cargo test --package e2e_test` - Run all e2e tests
- `./scripts/run_e2e_tests.sh` - Run e2e tests via script
- `./scripts/run_scanner_benchmarks.sh` - Run scanner performance benchmarks
### KMS-Specific Testing (with proxy bypass)
-
`NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1` -
Run complete KMS end-to-end test
-
`NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= cargo test --package e2e_test kms:: -- --nocapture --test-threads=1` -
Run all KMS tests
- `cargo test --package e2e_test test_local_kms_key_isolation -- --nocapture --test-threads=1` - Test KMS key isolation
- `cargo test --package e2e_test test_local_kms_large_file -- --nocapture --test-threads=1` - Test KMS with large files
### Code Quality
- `cargo fmt --all` - Format code
- `cargo clippy --all-targets --all-features -- -D warnings` - Lint code
- `make pre-commit` or `just pre-commit` - Run all quality checks (fmt, clippy, check, test)
### Quick Development Commands
- `make help` or `just help` - Show all available commands with descriptions
- `make help-build` - Show detailed build options and cross-compilation help
- `make help-docker` - Show comprehensive Docker build and deployment options
- `./scripts/dev_deploy.sh <IP>` - Deploy development build to remote server
- `./scripts/run.sh` - Start local development server
- `./scripts/probe.sh` - Health check and connectivity testing
### Docker Build Commands
- `make docker-buildx` - Build multi-architecture production images
- `make docker-dev-local` - Build development image for local use
- `./docker-buildx.sh --push` - Build and push production images
## Architecture Overview
### Core Components
**Main Binary (`rustfs/`):**
- Entry point at `rustfs/src/main.rs`
- Core modules: admin, auth, config, server, storage, license management, profiling
- HTTP server with S3-compatible APIs
- Service state management and graceful shutdown
- Parallel service initialization with DNS resolver, bucket metadata, and IAM
**Key Crates (`crates/`):**
- `ecstore` - Erasure coding storage implementation (core storage layer)
- `iam` - Identity and Access Management
- `kms` - Key Management Service for encryption and key handling
- `madmin` - Management dashboard and admin API interface
- `s3select-api` & `s3select-query` - S3 Select API and query engine
- `config` - Configuration management with notify features
- `crypto` - Cryptography and security features
- `lock` - Distributed locking implementation
- `filemeta` - File metadata management
- `rio` - Rust I/O utilities and abstractions
- `common` - Shared utilities and data structures
- `protos` - Protocol buffer definitions
- `audit-logger` - Audit logging for file operations
- `notify` - Event notification system
- `obs` - Observability utilities
- `workers` - Worker thread pools and task scheduling
- `appauth` - Application authentication and authorization
- `ahm` - Asynchronous Hash Map for concurrent data structures
- `mcp` - MCP server for S3 operations
- `signer` - Client request signing utilities
- `checksums` - Client checksum calculation utilities
- `utils` - General utility functions and helpers
- `zip` - ZIP file handling and compression
- `targets` - Target-specific configurations and utilities
### Build System
- Cargo workspace with 25+ crates (including new KMS functionality)
- Custom `build-rustfs.sh` script for advanced build options
- Multi-architecture Docker builds via `docker-buildx.sh`
- Both Make and Just task runners supported with comprehensive help
- Cross-compilation support for multiple Linux targets
- Automated CI/CD with GitHub Actions for testing, building, and Docker publishing
- Performance benchmarking and audit workflows
### Key Dependencies
- `axum` - HTTP framework for S3 API server
- `tokio` - Async runtime
- `s3s` - S3 protocol implementation library
- `datafusion` - For S3 Select query processing
- `hyper`/`hyper-util` - HTTP client/server utilities
- `rustls` - TLS implementation
- `serde`/`serde_json` - Serialization
- `tracing` - Structured logging and observability
- `pprof` - Performance profiling with flamegraph support
- `tikv-jemallocator` - Memory allocator for Linux GNU builds
### Development Workflow
- Console resources are embedded during build via `rust-embed`
- Protocol buffers generated via custom `gproto` binary
- E2E tests in separate crate (`e2e_test`) with comprehensive KMS testing
- Shadow build for version/metadata embedding
- Support for both GNU and musl libc targets
- Development scripts in `scripts/` directory for common tasks
- Git hooks setup available via `make setup-hooks` or `just setup-hooks`
### Performance & Observability
- Performance profiling available with `pprof` integration (disabled on Windows)
- Profiling enabled via environment variables in production
- Built-in observability with OpenTelemetry integration
- Background services (scanner, heal) can be controlled via environment variables:
- `RUSTFS_ENABLE_SCANNER` (default: true)
- `RUSTFS_ENABLE_HEAL` (default: true)
### Service Architecture
- Service state management with graceful shutdown handling
- Parallel initialization of core systems (DNS, bucket metadata, IAM)
- Event notification system with MQTT and webhook support
- Auto-heal and data scanner for storage integrity
- Jemalloc allocator for Linux GNU targets for better performance
## Environment Variables
- `RUSTFS_ENABLE_SCANNER` - Enable/disable background data scanner (default: true)
- `RUSTFS_ENABLE_HEAL` - Enable/disable auto-heal functionality (default: true)
- Various profiling and observability controls
- Build-time variables for Docker builds (RELEASE, REGISTRY, etc.)
- Test environment configurations in `scripts/dev_rustfs.env`
### KMS Environment Variables
- `NO_PROXY=127.0.0.1,localhost` - Required for KMS E2E tests to bypass proxy
- `HTTP_PROXY=` `HTTPS_PROXY=` `http_proxy=` `https_proxy=` - Clear proxy settings for local KMS testing
## KMS (Key Management Service) Architecture
### KMS Implementation Status
- **Full KMS Integration:** Complete implementation with Local and Vault backends
- **Automatic Configuration:** KMS auto-configures on startup with `--kms-enable` flag
- **Encryption Support:** Full S3-compatible server-side encryption (SSE-S3, SSE-KMS, SSE-C)
- **Admin API:** Complete KMS management via HTTP admin endpoints
- **Production Ready:** Comprehensive testing including large files and key isolation
### KMS Configuration
- **Local Backend:** `--kms-backend local --kms-key-dir <path> --kms-default-key-id <id>`
- **Vault Backend:** `--kms-backend vault --kms-vault-endpoint <url> --kms-vault-key-name <name>`
- **Auto-startup:** KMS automatically initializes when `--kms-enable` is provided
- **Manual Configuration:** Also supports dynamic configuration via admin API
### S3 Encryption Support
- **SSE-S3:** Server-side encryption with S3-managed keys (`ServerSideEncryption: AES256`)
- **SSE-KMS:** Server-side encryption with KMS-managed keys (`ServerSideEncryption: aws:kms`)
- **SSE-C:** Server-side encryption with customer-provided keys
- **Response Headers:** All encryption types return correct `server_side_encryption` headers in PUT/GET responses
### KMS Testing Architecture
- **Comprehensive E2E Tests:** Located in `crates/e2e_test/src/kms/`
- **Test Environments:** Automated test environment setup with temporary directories
- **Encryption Coverage:** Tests all three encryption types (SSE-S3, SSE-KMS, SSE-C)
- **API Coverage:** Tests all KMS admin APIs (CreateKey, DescribeKey, ListKeys, etc.)
- **Edge Cases:** Key isolation, large file handling, error scenarios
### Key Files for KMS
- `crates/kms/` - Core KMS implementation with Local/Vault backends
- `rustfs/src/main.rs` - KMS auto-initialization in `init_kms_system()`
- `rustfs/src/storage/ecfs.rs` - SSE encryption/decryption in PUT/GET operations
- `rustfs/src/admin/handlers/kms*.rs` - KMS admin endpoints
- `crates/e2e_test/src/kms/` - Comprehensive KMS test suite
- `crates/rio/src/encrypt_reader.rs` - Streaming encryption for large files
## Code Style and Safety Requirements
- **Language Requirements:**
- Communicate with me in Chinese, but **only English can be used in code files**
- Code comments, function names, variable names, and all text in source files must be in English only
- No Chinese characters, emojis, or non-ASCII characters are allowed in any source code files
- This includes comments, strings, documentation, and any other text within code files
- **Safety-Critical Rules:**
- `unsafe_code = "deny"` enforced at workspace level
- Never use `unwrap()`, `expect()`, or panic-inducing code except in tests
- Avoid blocking I/O operations in async contexts
- Use proper error handling with `Result<T, E>` and `Option<T>`
- Follow Rust's ownership and borrowing rules strictly
- **Performance Guidelines:**
- Use `cargo clippy --all-targets --all-features -- -D warnings` to catch issues
- Prefer `anyhow` for error handling in applications, `thiserror` for libraries
- Use appropriate async runtimes and avoid blocking calls
- **Testing Standards:**
- All new features must include comprehensive tests
- Use `#[cfg(test)]` for test-only code that may use panic macros
- E2E tests should cover KMS integration scenarios
## Common Development Tasks
### Running KMS Tests Locally
1. **Clear proxy settings:** KMS tests require direct localhost connections
2. **Use serial execution:** `--test-threads=1` prevents port conflicts
3. **Enable output:** `--nocapture` shows detailed test logs
4. **Full command:**
`NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1`
### KMS Development Workflow
1. **Code changes:** Modify KMS-related code in `crates/kms/` or `rustfs/src/`
2. **Compile:** Always run `cargo build` after changes
3. **Test specific functionality:** Use targeted test commands for faster iteration
4. **Full validation:** Run complete end-to-end tests before commits
### Debugging KMS Issues
- **Server startup:** Check that KMS auto-initializes with debug logs
- **Encryption failures:** Verify SSE headers are correctly set in both PUT and GET responses
- **Test failures:** Use `--nocapture` to see detailed error messages
- **Key management:** Test admin API endpoints with proper authentication
## Important Reminders
- **Always compile after code changes:** Use `cargo build` to catch errors early
- **Don't bypass tests:** All functionality must be properly tested, not worked around
- **Use proper error handling:** Never use `unwrap()` or `expect()` in production code (except tests)
- **Follow S3 compatibility:** Ensure all encryption types return correct HTTP response headers
# important-instruction-reminders
Do what has been asked; nothing more, nothing less.
NEVER create files unless they're absolutely necessary for achieving your goal.
ALWAYS prefer editing an existing file to creating a new one.
NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly
requested by the User.
-35
View File
@@ -2,8 +2,6 @@
## 📋 Code Quality Requirements
For instructions on setting up and running the local development environment, please see [Development Guide](docs/DEVELOPMENT.md).
### 🔧 Code Formatting Rules
**MANDATORY**: All code must be properly formatted before committing. This project enforces strict formatting standards to maintain code consistency and readability.
@@ -186,39 +184,6 @@ cargo clippy --all-targets --all-features -- -D warnings
cargo clippy --fix --all-targets --all-features
```
## 📝 Pull Request Guidelines
### Language Requirements
**All Pull Request titles and descriptions MUST be written in English.**
This ensures:
- Consistency across all contributions
- Accessibility for international contributors
- Better integration with automated tools and CI/CD systems
- Clear communication in a globally understood language
#### PR Description Requirements
When creating a Pull Request, ensure:
1. **Title**: Use English and follow Conventional Commits format (e.g., `fix: improve s3-tests readiness detection`)
2. **Description**: Write in English, following the PR template format
3. **Code Comments**: Must be in English (as per coding standards)
4. **Commit Messages**: Must be in English (as per commit guidelines)
#### PR Template
Always use the PR template (`.github/pull_request_template.md`) and fill in all sections:
- Type of Change
- Related Issues
- Summary of Changes
- Checklist
- Impact
- Additional Notes
**Note**: While you may communicate with reviewers in Chinese during discussions, the PR itself (title, description, and all formal documentation) must be in English.
---
Following these guidelines ensures high code quality and smooth collaboration across the RustFS project! 🚀
Generated
+2132 -2422
View File
File diff suppressed because it is too large Load Diff
+111 -123
View File
@@ -17,36 +17,31 @@ members = [
"rustfs", # Core file system implementation
"crates/appauth", # Application authentication and authorization
"crates/audit", # Audit target management system with multi-target fan-out
"crates/checksums", # client checksums
"crates/common", # Shared utilities and data structures
"crates/config", # Configuration management
"crates/credentials", # Credential management system
"crates/crypto", # Cryptography and security features
"crates/ecstore", # Erasure coding storage implementation
"crates/e2e_test", # End-to-end test suite
"crates/filemeta", # File metadata management
"crates/heal", # Erasure set and object healing
"crates/iam", # Identity and Access Management
"crates/kms", # Key Management Service
"crates/lock", # Distributed locking implementation
"crates/madmin", # Management dashboard and admin API interface
"crates/mcp", # MCP server for S3 operations
"crates/metrics", # Metrics collection and reporting
"crates/notify", # Notification system for events
"crates/obs", # Observability utilities
"crates/policy", # Policy management
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
"crates/protos", # Protocol buffer definitions
"crates/rio", # Rust I/O utilities and abstractions
"crates/targets", # Target-specific configurations and utilities
"crates/s3select-api", # S3 Select API interface
"crates/s3select-query", # S3 Select query engine
"crates/scanner", # Scanner for data integrity checks and health monitoring
"crates/signer", # client signer
"crates/targets", # Target-specific configurations and utilities
"crates/trusted-proxies", # Trusted proxies management
"crates/checksums", # client checksums
"crates/utils", # Utility functions and helpers
"crates/workers", # Worker thread pools and task scheduling
"crates/zip", # ZIP file handling and compression
"crates/ahm", # Asynchronous Hash Map for concurrent data structures
"crates/mcp", # MCP server for S3 operations
"crates/kms", # Key Management Service
]
resolver = "2"
@@ -54,13 +49,12 @@ resolver = "2"
edition = "2024"
license = "Apache-2.0"
repository = "https://github.com/rustfs/rustfs"
rust-version = "1.93.0"
rust-version = "1.85"
version = "0.0.5"
homepage = "https://rustfs.com"
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
categories = ["web-programming", "development-tools", "filesystem", "network-programming"]
authors = ["RustFS Team"]
[workspace.lints.rust]
unsafe_code = "deny"
@@ -71,13 +65,12 @@ all = "warn"
[workspace.dependencies]
# RustFS Internal Crates
rustfs = { path = "./rustfs", version = "0.0.5" }
rustfs-heal = { path = "crates/heal", version = "0.0.5" }
rustfs-ahm = { path = "crates/ahm", version = "0.0.5" }
rustfs-appauth = { path = "crates/appauth", version = "0.0.5" }
rustfs-audit = { path = "crates/audit", version = "0.0.5" }
rustfs-checksums = { path = "crates/checksums", version = "0.0.5" }
rustfs-common = { path = "crates/common", version = "0.0.5" }
rustfs-config = { path = "./crates/config", version = "0.0.5" }
rustfs-credentials = { path = "crates/credentials", version = "0.0.5" }
rustfs-crypto = { path = "crates/crypto", version = "0.0.5" }
rustfs-ecstore = { path = "crates/ecstore", version = "0.0.5" }
rustfs-filemeta = { path = "crates/filemeta", version = "0.0.5" }
@@ -86,7 +79,6 @@ rustfs-kms = { path = "crates/kms", version = "0.0.5" }
rustfs-lock = { path = "crates/lock", version = "0.0.5" }
rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
rustfs-metrics = { path = "crates/metrics", version = "0.0.5" }
rustfs-notify = { path = "crates/notify", version = "0.0.5" }
rustfs-obs = { path = "crates/obs", version = "0.0.5" }
rustfs-policy = { path = "crates/policy", version = "0.0.5" }
@@ -94,205 +86,201 @@ rustfs-protos = { path = "crates/protos", version = "0.0.5" }
rustfs-rio = { path = "crates/rio", version = "0.0.5" }
rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.5" }
rustfs-scanner = { path = "crates/scanner", version = "0.0.5" }
rustfs-signer = { path = "crates/signer", version = "0.0.5" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "0.0.5" }
rustfs-targets = { path = "crates/targets", version = "0.0.5" }
rustfs-utils = { path = "crates/utils", version = "0.0.5" }
rustfs-workers = { path = "crates/workers", version = "0.0.5" }
rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
# Async Runtime and Networking
async-channel = "2.5.0"
async-compression = { version = "0.4.39" }
async-compression = { version = "0.4.19" }
async-recursion = "1.1.1"
async-trait = "0.1.89"
axum = "0.8.8"
axum-server = { version = "0.8.0", features = ["tls-rustls"], default-features = false }
axum = "0.8.6"
axum-extra = "0.12.1"
axum-server = { version = "0.7.2", features = ["tls-rustls-no-provider"], default-features = false }
futures = "0.3.31"
futures-core = "0.3.31"
futures-util = "0.3.31"
pollster = "0.4.0"
hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
http = "1.4.0"
hyper = { version = "1.7.0", features = ["http2", "http1", "server"] }
hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "ring", "webpki-roots"] }
hyper-util = { version = "0.1.17", features = ["tokio", "server-auto", "server-graceful"] }
http = "1.3.1"
http-body = "1.0.1"
http-body-util = "0.1.3"
reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
socket2 = { version = "0.6.2", features = ["all"] }
tokio = { version = "1.49.0", features = ["fs", "rt-multi-thread"] }
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
tokio-stream = { version = "0.1.18" }
tokio-test = "0.4.5"
tokio-util = { version = "0.7.18", features = ["io", "compat"] }
tonic = { version = "0.14.3", features = ["gzip"] }
tonic-prost = { version = "0.14.3" }
tonic-prost-build = { version = "0.14.3" }
tower = { version = "0.5.3", features = ["timeout"] }
tower-http = { version = "0.6.8", features = ["cors"] }
reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2", "system-proxy", "stream", "json", "blocking"] }
socket2 = "0.6.1"
tokio = { version = "1.48.0", features = ["fs", "rt-multi-thread"] }
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "ring"] }
tokio-stream = { version = "0.1.17" }
tokio-test = "0.4.4"
tokio-util = { version = "0.7.17", features = ["io", "compat"] }
tonic = { version = "0.14.2", features = ["gzip"] }
tonic-prost = { version = "0.14.2" }
tonic-prost-build = { version = "0.14.2" }
tower = { version = "0.5.2", features = ["timeout"] }
tower-http = { version = "0.6.6", features = ["cors"] }
# Serialization and Data Formats
bytes = { version = "1.11.1", features = ["serde"] }
bytesize = "2.3.1"
bytes = { version = "1.10.1", features = ["serde"] }
bytesize = "2.1.0"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
flatbuffers = "25.9.23"
form_urlencoded = "1.2.2"
prost = "0.14.3"
quick-xml = "0.39.0"
rmcp = { version = "0.15.0" }
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
prost = "0.14.1"
quick-xml = "0.38.3"
rmcp = { version = "0.8.4" }
rmp = { version = "0.8.14" }
rmp-serde = { version = "1.3.0" }
serde = { version = "1.0.228", features = ["derive"] }
serde_json = { version = "1.0.149", features = ["raw_value"] }
serde_json = { version = "1.0.145", features = ["raw_value"] }
serde_urlencoded = "0.7.1"
schemars = "1.2.1"
schemars = "1.0.5"
# Cryptography and Security
aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
argon2 = { version = "0.6.0-rc.7" }
blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
chacha20poly1305 = { version = "0.11.0-rc.3" }
crc-fast = "1.9.0"
hmac = { version = "0.13.0-rc.5" }
jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
pbkdf2 = "0.13.0-rc.9"
rsa = { version = "0.10.0-rc.15" }
rustls = { version = "0.23.36", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
aes-gcm = { version = "0.10.3", features = ["std"] }
argon2 = { version = "0.5.3", features = ["std"] }
blake3 = { version = "1.8.2" }
chacha20poly1305 = { version = "0.10.1" }
crc-fast = "1.3.0"
crc32c = "0.6.8"
crc32fast = "1.5.0"
crc64fast-nvme = "1.2.0"
hmac = "0.12.1"
jsonwebtoken = { version = "10.1.0", features = ["rust_crypto"] }
pbkdf2 = "0.12.2"
rsa = { version = "0.9.8" }
rustls = { version = "0.23.35", features = ["ring", "logging", "std", "tls12"], default-features = false }
rustls-pemfile = "2.2.0"
rustls-pki-types = "1.14.0"
sha1 = "0.11.0-rc.5"
sha2 = "0.11.0-rc.5"
subtle = "2.6"
rustls-pki-types = "1.13.0"
sha1 = "0.10.6"
sha2 = "0.10.9"
zeroize = { version = "1.8.2", features = ["derive"] }
# Time and Date
chrono = { version = "0.4.43", features = ["serde"] }
chrono = { version = "0.4.42", features = ["serde"] }
humantime = "2.3.0"
jiff = { version = "0.2.19", features = ["serde"] }
time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
time = { version = "0.3.44", features = ["std", "parsing", "formatting", "macros", "serde"] }
# Utilities and Tools
anyhow = "1.0.101"
arc-swap = "1.8.1"
anyhow = "1.0.100"
arc-swap = "1.7.1"
astral-tokio-tar = "0.5.6"
atoi = "2.0.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.8.13" }
aws-credential-types = { version = "1.2.11" }
aws-sdk-s3 = { version = "1.122.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-smithy-types = { version = "1.4.4" }
backtrace = "0.3.76"
aws-config = { version = "1.8.10" }
aws-credential-types = { version = "1.2.8" }
aws-sdk-s3 = { version = "1.110.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] }
aws-smithy-types = { version = "1.3.4" }
base64 = "0.22.1"
base64-simd = "0.8.0"
brotli = "8.0.2"
cfg-if = "1.0.4"
clap = { version = "4.5.57", features = ["derive", "env"] }
const-str = { version = "1.1.0", features = ["std", "proc"] }
convert_case = "0.11.0"
criterion = { version = "0.8", features = ["html_reports"] }
clap = { version = "4.5.51", features = ["derive", "env"] }
const-str = { version = "0.7.0", features = ["std", "proc"] }
convert_case = "0.8.0"
criterion = { version = "0.7", features = ["html_reports"] }
crossbeam-queue = "0.3.12"
datafusion = "52.1.0"
datafusion = "50.3.0"
derive_builder = "0.20.2"
enumset = "1.1.10"
faster-hex = "0.10.0"
flate2 = "1.1.9"
flexi_logger = { version = "0.31.8", features = ["trc", "dont_minimize_extra_stacks", "compress", "kv", "json"] }
flate2 = "1.1.5"
flexi_logger = { version = "0.31.7", features = ["trc", "dont_minimize_extra_stacks", "compress", "kv"] }
glob = "0.3.3"
google-cloud-storage = "1.7.0"
google-cloud-auth = "1.5.0"
hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
google-cloud-storage = "1.2.0"
google-cloud-auth = "1.1.0"
hashbrown = { version = "0.16.0", features = ["serde", "rayon"] }
heed = { version = "0.22.0" }
hex-simd = "0.8.0"
highway = { version = "1.3.0" }
ipnetwork = { version = "0.21.1", features = ["serde"] }
lazy_static = "1.5.0"
libc = "0.2.181"
libsystemd = "0.7.2"
local-ip-address = "0.6.10"
libc = "0.2.177"
libsystemd = { version = "0.7.2" }
local-ip-address = "0.6.5"
lz4 = "1.28.1"
matchit = "0.9.1"
md-5 = "0.11.0-rc.5"
matchit = "0.9.0"
md-5 = "0.10.6"
md5 = "0.8.0"
metrics = "0.24.2"
metrics-exporter-opentelemetry = "0.1.2"
mime_guess = "2.0.5"
moka = { version = "0.12.13", features = ["future"] }
moka = { version = "0.12.11", features = ["future"] }
netif = "0.1.6"
nix = { version = "0.30.1", features = ["fs"] }
nu-ansi-term = "0.50.3"
num_cpus = { version = "1.17.0" }
nvml-wrapper = "0.11.0"
object_store = "0.12.5"
object_store = "0.12.4"
once_cell = "1.21.3"
parking_lot = "0.12.5"
path-absolutize = "3.1.1"
path-clean = "1.0.1"
pin-project-lite = "0.2.16"
pretty_assertions = "1.4.1"
rand = { version = "0.10.0", features = ["serde"] }
rand = "0.9.2"
rayon = "1.11.0"
reed-solomon-simd = { version = "3.1.0" }
regex = { version = "1.12.3" }
rumqttc = { version = "0.25.1" }
rustix = { version = "1.1.3", features = ["fs"] }
rust-embed = { version = "8.11.0" }
regex = { version = "1.12.2" }
rumqttc = { version = "0.25.0" }
rust-embed = { version = "8.9.0" }
rustc-hash = { version = "2.1.1" }
s3s = { version = "0.13.0-alpha.3", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", rev = "7b8e22b49bc76839e6085a4ddfa553323f1b8049" }
serial_test = "3.3.1"
shadow-rs = { version = "1.7.0", default-features = false }
siphasher = "1.0.2"
s3s = { version = "0.12.0-rc.3", features = ["minio"] }
scopeguard = "1.2.0"
serial_test = "3.2.0"
shadow-rs = { version = "1.4.0", default-features = false }
siphasher = "1.0.1"
smallvec = { version = "1.15.1", features = ["serde"] }
smartstring = "1.0.1"
snafu = "0.8.9"
snap = "1.1.1"
starshard = { version = "1.1.0", features = ["rayon", "async", "serde"] }
starshard = { version = "0.5.0", features = ["rayon", "async", "serde"] }
strum = { version = "0.27.2", features = ["derive"] }
sysinfo = "0.38.1"
sysctl = "0.7.1"
sysinfo = "0.37.2"
temp-env = "0.3.6"
tempfile = "3.25.0"
tempfile = "3.23.0"
test-case = "3.3.1"
thiserror = "2.0.18"
tracing = { version = "0.1.44" }
tracing-appender = "0.2.4"
thiserror = "2.0.17"
tracing = { version = "0.1.41" }
tracing-appender = "0.2.3"
tracing-error = "0.2.1"
tracing-opentelemetry = "0.32.1"
tracing-subscriber = { version = "0.3.22", features = ["env-filter", "time"] }
tracing-opentelemetry = "0.32.0"
tracing-subscriber = { version = "0.3.20", features = ["env-filter", "time"] }
transform-stream = "0.3.1"
url = "2.5.8"
url = "2.5.7"
urlencoding = "2.1.3"
uuid = { version = "1.20.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
uuid = { version = "1.18.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
vaultrs = { version = "0.7.4" }
walkdir = "2.5.0"
wildmatch = { version = "2.6.1", features = ["serde"] }
windows = { version = "0.62.2" }
wildmatch = { version = "2.5.0", features = ["serde"] }
winapi = { version = "0.3.9" }
xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
zip = "7.4.0"
zip = "6.0.0"
zstd = "0.13.3"
# Observability and Metrics
metrics = "0.24.3"
opentelemetry = { version = "0.31.0" }
opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
opentelemetry-otlp = { version = "0.31.0", features = ["gzip-http", "reqwest-rustls"] }
opentelemetry-otlp = { version = "0.31.0", default-features = false, features = ["grpc-tonic", "gzip-tonic", "trace", "metrics", "logs", "internal-logs"] }
opentelemetry_sdk = { version = "0.31.0" }
opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
opentelemetry-stdout = { version = "0.31.0" }
# FTP and SFTP
libunftp = { version = "0.22.0", features = ["experimental"] }
suppaftp = { version = "8.0.1", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
rcgen = "0.14.7"
# Performance Analysis and Memory Profiling
mimalloc = "0.1"
# Use tikv-jemallocator as memory allocator and enable performance analysis
tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unprefixed_malloc_on_supported_platforms", "background_threads"] }
# Used to control and obtain statistics for jemalloc at runtime
tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats", "profiling"] }
# Used to generate pprof-compatible memory profiling data and support symbolization and flame graphs
jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
jemalloc_pprof = { version = "0.8.1", features = ["symbolize", "flamegraph"] }
# Used to generate CPU performance analysis data and flame diagrams
pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] }
mimalloc = "0.1"
[workspace.metadata.cargo-shear]
ignored = ["rustfs", "rustfs-mcp"]
ignored = ["rustfs", "rustfs-mcp", "tokio-test", "scopeguard"]
[profile.release]
opt-level = 3
+9 -15
View File
@@ -1,4 +1,4 @@
FROM alpine:3.23 AS build
FROM alpine:3.22 AS build
ARG TARGETARCH
ARG RELEASE=latest
@@ -40,7 +40,7 @@ RUN set -eux; \
rm -rf rustfs.zip /build/.tmp || true
FROM alpine:3.23
FROM alpine:3.22
ARG RELEASE=latest
ARG BUILD_DATE
@@ -64,12 +64,8 @@ COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=build /build/rustfs /usr/bin/rustfs
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
RUN addgroup -g 10001 -S rustfs && \
adduser -u 10001 -G rustfs -S rustfs -D && \
RUN chmod +x /usr/bin/rustfs /entrypoint.sh && \
mkdir -p /data /logs && \
chown -R rustfs:rustfs /data /logs && \
chmod 0750 /data /logs
ENV RUSTFS_ADDRESS=":9000" \
@@ -77,18 +73,16 @@ ENV RUSTFS_ADDRESS=":9000" \
RUSTFS_ACCESS_KEY="rustfsadmin" \
RUSTFS_SECRET_KEY="rustfsadmin" \
RUSTFS_CONSOLE_ENABLE="true" \
RUSTFS_EXTERNAL_ADDRESS="" \
RUSTFS_CORS_ALLOWED_ORIGINS="*" \
RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS="*" \
RUSTFS_VOLUMES="/data" \
RUSTFS_OBS_LOGGER_LEVEL=warn \
RUSTFS_OBS_LOG_DIRECTORY=/logs \
RUSTFS_OBS_ENVIRONMENT=production
RUST_LOG="warn" \
RUSTFS_OBS_LOG_DIRECTORY="/logs" \
RUSTFS_SINKS_FILE_PATH="/logs"
EXPOSE 9000 9001
VOLUME ["/data"]
USER rustfs
VOLUME ["/data", "/logs"]
ENTRYPOINT ["/entrypoint.sh"]
-101
View File
@@ -1,101 +0,0 @@
FROM ubuntu:24.04 AS build
ARG TARGETARCH
ARG RELEASE=latest
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
curl \
unzip \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /build
RUN set -eux; \
case "$TARGETARCH" in \
amd64) ARCH_SUBSTR="x86_64-gnu" ;; \
arm64) ARCH_SUBSTR="aarch64-gnu" ;; \
*) echo "Unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;; \
esac; \
\
if [ "$RELEASE" = "latest" ]; then \
TAG="$(curl -fsSL https://api.github.com/repos/rustfs/rustfs/releases \
| grep -o '"tag_name": "[^"]*"' | cut -d'"' -f4 | head -n 1)"; \
else \
TAG="$RELEASE"; \
fi; \
\
URL="$(curl -fsSL "https://api.github.com/repos/rustfs/rustfs/releases/tags/$TAG" \
| grep -o "\"browser_download_url\": \"[^\"]*${ARCH_SUBSTR}[^\"]*\\.zip\"" \
| cut -d'"' -f4 | head -n 1)"; \
\
if [ -z "$URL" ]; then echo "Failed to locate release asset for $ARCH_SUBSTR at tag $TAG" >&2; exit 1; fi; \
\
curl -fL "$URL" -o rustfs.zip; \
unzip -q rustfs.zip -d /build; \
\
if [ ! -x /build/rustfs ]; then \
BIN_PATH="$(unzip -Z -1 rustfs.zip | grep -E '(^|/)rustfs$' | head -n 1 || true)"; \
if [ -n "$BIN_PATH" ]; then \
mkdir -p /build/.tmp && unzip -q rustfs.zip "$BIN_PATH" -d /build/.tmp && \
mv "/build/.tmp/$BIN_PATH" /build/rustfs; \
fi; \
fi; \
[ -x /build/rustfs ] || { echo "rustfs binary not found in asset" >&2; exit 1; }; \
chmod +x /build/rustfs; \
rm -rf rustfs.zip /build/.tmp || true
FROM ubuntu:24.04
ARG RELEASE=latest
ARG BUILD_DATE
ARG VCS_REF
LABEL name="RustFS" \
vendor="RustFS Team" \
maintainer="RustFS Team <dev@rustfs.com>" \
version="v${RELEASE#v}" \
release="${RELEASE}" \
build-date="${BUILD_DATE}" \
vcs-ref="${VCS_REF}" \
summary="High-performance distributed object storage system (glibc)" \
url="https://rustfs.com" \
license="Apache-2.0"
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
curl \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /build/rustfs /usr/bin/rustfs
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
RUN groupadd -g 10001 rustfs && \
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
mkdir -p /data /logs && \
chown -R rustfs:rustfs /data /logs && \
chmod 0750 /data /logs
ENV RUSTFS_ADDRESS=":9000" \
RUSTFS_CONSOLE_ADDRESS=":9001" \
RUSTFS_ACCESS_KEY="rustfsadmin" \
RUSTFS_SECRET_KEY="rustfsadmin" \
RUSTFS_CONSOLE_ENABLE="true" \
RUSTFS_CORS_ALLOWED_ORIGINS="*" \
RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS="*" \
RUSTFS_VOLUMES="/data" \
RUSTFS_OBS_LOGGER_LEVEL=warn \
RUSTFS_OBS_LOG_DIRECTORY=/logs \
RUSTFS_OBS_ENVIRONMENT=production
EXPOSE 9000 9001
VOLUME ["/data"]
USER rustfs
ENTRYPOINT ["/entrypoint.sh"]
CMD ["rustfs"]
+16 -62
View File
@@ -16,7 +16,7 @@ ARG BUILDPLATFORM
# -----------------------------
# Build stage
# -----------------------------
FROM rust:1.91-trixie AS builder
FROM rust:1.88-bookworm AS builder
# Re-declare args after FROM
ARG TARGETPLATFORM
@@ -39,9 +39,7 @@ RUN set -eux; \
libssl-dev \
lld \
protobuf-compiler \
flatbuffers-compiler \
gcc-aarch64-linux-gnu \
gcc-x86-64-linux-gnu; \
flatbuffers-compiler; \
rm -rf /var/lib/apt/lists/*
# Optional: cross toolchain for aarch64 (only when targeting linux/arm64)
@@ -53,18 +51,18 @@ RUN set -eux; \
rm -rf /var/lib/apt/lists/*; \
fi
# Add Rust targets for both arches (to support cross-builds on multi-arch runners)
# Add Rust targets based on TARGETPLATFORM
RUN set -eux; \
rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; \
rustup component add rust-std-x86_64-unknown-linux-gnu rust-std-aarch64-unknown-linux-gnu
case "${TARGETPLATFORM:-linux/amd64}" in \
linux/amd64) rustup target add x86_64-unknown-linux-gnu ;; \
linux/arm64) rustup target add aarch64-unknown-linux-gnu ;; \
*) echo "Unsupported TARGETPLATFORM=${TARGETPLATFORM}" >&2; exit 1 ;; \
esac
# Cross-compilation environment (used only when targeting aarch64)
ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
ENV CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc
ENV CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++
ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc
ENV CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc
ENV CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++
WORKDIR /usr/src/rustfs
@@ -74,6 +72,7 @@ COPY Cargo.toml Cargo.lock ./
# 2) workspace member manifests (adjust if workspace layout changes)
COPY rustfs/Cargo.toml rustfs/Cargo.toml
COPY crates/*/Cargo.toml crates/
COPY cli/rustfs-gui/Cargo.toml cli/rustfs-gui/Cargo.toml
# Pre-fetch dependencies for better caching
RUN --mount=type=cache,target=/usr/local/cargo/registry \
@@ -83,10 +82,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
# 3) copy full sources (this is the main cache invalidation point)
COPY . .
# Generate static files
RUN ./scripts/static.sh
# Cargo build configuration for lean release artifacts
ENV CARGO_NET_GIT_FETCH_WITH_CLI=true \
CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse \
@@ -122,49 +117,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
;; \
esac
# -----------------------------
# Development stage (keeps toolchain)
# -----------------------------
FROM builder AS dev
ARG BUILD_DATE
ARG VCS_REF
LABEL name="RustFS (dev-source)" \
maintainer="RustFS Team" \
build-date="${BUILD_DATE}" \
vcs-ref="${VCS_REF}" \
description="RustFS - local development with Rust toolchain."
# Install runtime dependencies that might be missing in partial builder
# (builder already has build-essential, lld, etc.)
WORKDIR /app
ENV CARGO_INCREMENTAL=1
# Ensure we have the same default env vars available
ENV RUSTFS_ADDRESS=":9000" \
RUSTFS_ACCESS_KEY="rustfsadmin" \
RUSTFS_SECRET_KEY="rustfsadmin" \
RUSTFS_CONSOLE_ENABLE="true" \
RUSTFS_VOLUMES="/data" \
RUST_LOG="warn" \
RUSTFS_OBS_LOG_DIRECTORY="/logs" \
RUSTFS_USERNAME="rustfs" \
RUSTFS_GROUPNAME="rustfs" \
RUSTFS_UID="10001" \
RUSTFS_GID="10001"
# Note: We don't COPY source here because we expect it to be mounted at /app
# We rely on cargo run to build and run
EXPOSE 9000 9001
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
CMD ["cargo", "run", "--bin", "rustfs", "--"]
# -----------------------------
# Runtime stage (Ubuntu minimal)
# -----------------------------
@@ -191,8 +143,8 @@ RUN set -eux; \
# Create a conventional runtime user/group (final switch happens in entrypoint via chroot --userspec)
RUN set -eux; \
groupadd -g 10001 rustfs; \
useradd -u 10001 -g rustfs -M -s /usr/sbin/nologin rustfs
groupadd -g 1000 rustfs; \
useradd -u 1000 -g rustfs -M -s /usr/sbin/nologin rustfs
WORKDIR /app
@@ -214,13 +166,15 @@ ENV RUSTFS_ADDRESS=":9000" \
RUSTFS_CONSOLE_ENABLE="true" \
RUSTFS_VOLUMES="/data" \
RUST_LOG="warn" \
RUSTFS_OBS_LOG_DIRECTORY="/logs" \
RUSTFS_SINKS_FILE_PATH="/logs" \
RUSTFS_USERNAME="rustfs" \
RUSTFS_GROUPNAME="rustfs" \
RUSTFS_UID="10001" \
RUSTFS_GID="10001"
RUSTFS_UID="1000" \
RUSTFS_GID="1000"
EXPOSE 9000
VOLUME ["/data"]
VOLUME ["/data", "/logs"]
# Keep root here; entrypoint will drop privileges using chroot --userspec
ENTRYPOINT ["/entrypoint.sh"]
+353 -58
View File
@@ -2,80 +2,375 @@
# Remote development requires VSCode with Dev Containers, Remote SSH, Remote Explorer
# https://code.visualstudio.com/docs/remote/containers
###########
.PHONY: SHELL
# Makefile global config
# Use config.mak to override any of the following variables.
# Do not make changes here.
.DEFAULT_GOAL := help
.EXPORT_ALL_VARIABLES:
.ONESHELL:
.SILENT:
NUM_CORES := $(shell nproc 2>/dev/null || sysctl -n hw.ncpu)
MAKEFLAGS += -j$(NUM_CORES) -l$(NUM_CORES)
MAKEFLAGS += --silent
SHELL := $(shell which bash)
.SHELLFLAGS = -eu -o pipefail -c
DOCKER_CLI ?= docker
IMAGE_NAME ?= rustfs:v1.0.0
CONTAINER_NAME ?= rustfs-dev
# Docker build configurations
DOCKERFILE_PRODUCTION = Dockerfile
DOCKERFILE_SOURCE = Dockerfile.source
# Code quality and formatting targets
.PHONY: fmt
fmt:
@echo "🔧 Formatting code..."
cargo fmt --all
.PHONY: fmt-check
fmt-check:
@echo "📝 Checking code formatting..."
cargo fmt --all --check
.PHONY: clippy
clippy:
@echo "🔍 Running clippy checks..."
cargo clippy --fix --allow-dirty
cargo clippy --all-targets --all-features -- -D warnings
.PHONY: check
check:
@echo "🔨 Running compilation check..."
cargo check --all-targets
.PHONY: test
test:
@echo "🧪 Running tests..."
@if command -v cargo-nextest >/dev/null 2>&1; then \
cargo nextest run --all --exclude e2e_test; \
else \
echo "️ cargo-nextest not found; falling back to 'cargo test'"; \
cargo test --workspace --exclude e2e_test -- --nocapture; \
fi
cargo test --all --doc
.PHONY: pre-commit
pre-commit: fmt clippy check test
@echo "✅ All pre-commit checks passed!"
.PHONY: setup-hooks
setup-hooks:
@echo "🔧 Setting up git hooks..."
chmod +x .git/hooks/pre-commit
@echo "✅ Git hooks setup complete!"
.PHONY: e2e-server
e2e-server:
sh $(shell pwd)/scripts/run.sh
.PHONY: probe-e2e
probe-e2e:
sh $(shell pwd)/scripts/probe.sh
# Native build using build-rustfs.sh script
.PHONY: build
build:
@echo "🔨 Building RustFS using build-rustfs.sh script..."
./build-rustfs.sh
.PHONY: build-dev
build-dev:
@echo "🔨 Building RustFS in development mode..."
./build-rustfs.sh --dev
# Docker-based build (alternative approach)
# Usage: make BUILD_OS=ubuntu22.04 build-docker
# Output: target/ubuntu22.04/release/rustfs
BUILD_OS ?= rockylinux9.3
.PHONY: build-docker
build-docker: SOURCE_BUILD_IMAGE_NAME = rustfs-$(BUILD_OS):v1
build-docker: SOURCE_BUILD_CONTAINER_NAME = rustfs-$(BUILD_OS)-build
build-docker: BUILD_CMD = /root/.cargo/bin/cargo build --release --bin rustfs --target-dir /root/s3-rustfs/target/$(BUILD_OS)
build-docker:
@echo "🐳 Building RustFS using Docker ($(BUILD_OS))..."
$(DOCKER_CLI) buildx build -t $(SOURCE_BUILD_IMAGE_NAME) -f $(DOCKERFILE_SOURCE) .
$(DOCKER_CLI) run --rm --name $(SOURCE_BUILD_CONTAINER_NAME) -v $(shell pwd):/root/s3-rustfs -it $(SOURCE_BUILD_IMAGE_NAME) $(BUILD_CMD)
# Makefile colors config
bold := $(shell tput bold)
normal := $(shell tput sgr0)
errorTitle := $(shell tput setab 1 && tput bold && echo '\n')
recommendation := $(shell tput setab 4)
underline := $(shell tput smul)
reset := $(shell tput -Txterm sgr0)
black := $(shell tput setaf 0)
red := $(shell tput setaf 1)
green := $(shell tput setaf 2)
yellow := $(shell tput setaf 3)
blue := $(shell tput setaf 4)
magenta := $(shell tput setaf 5)
cyan := $(shell tput setaf 6)
white := $(shell tput setaf 7)
.PHONY: build-musl
build-musl:
@echo "🔨 Building rustfs for x86_64-unknown-linux-musl..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform x86_64-unknown-linux-musl
define HEADER
How to use me:
# To get help for each target
${bold}make help${reset}
.PHONY: build-gnu
build-gnu:
@echo "🔨 Building rustfs for x86_64-unknown-linux-gnu..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform x86_64-unknown-linux-gnu
# To run and execute a target
${bold}make ${cyan}<target>${reset}
.PHONY: build-musl-arm64
build-musl-arm64:
@echo "🔨 Building rustfs for aarch64-unknown-linux-musl..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform aarch64-unknown-linux-musl
💡 For more help use 'make help', 'make help-build' or 'make help-docker'
.PHONY: build-gnu-arm64
build-gnu-arm64:
@echo "🔨 Building rustfs for aarch64-unknown-linux-gnu..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform aarch64-unknown-linux-gnu
🦀 RustFS Makefile Help:
.PHONY: deploy-dev
deploy-dev: build-musl
@echo "🚀 Deploying to dev server: $${IP}"
./scripts/dev_deploy.sh $${IP}
📋 Main Command Categories:
make help-build # Show build-related help
make help-docker # Show Docker-related help
# ========================================================================================
# Docker Multi-Architecture Builds (Primary Methods)
# ========================================================================================
🔧 Code Quality:
make fmt # Format code
make clippy # Run clippy checks
make test # Run tests
make pre-commit # Run all pre-commit checks
# Production builds using docker-buildx.sh (for CI/CD and production)
.PHONY: docker-buildx
docker-buildx:
@echo "🏗️ Building multi-architecture production Docker images with buildx..."
./docker-buildx.sh
🚀 Quick Start:
make build # Build RustFS binary
make docker-dev-local # Build development Docker image (local)
make dev-env-start # Start development environment
.PHONY: docker-buildx-push
docker-buildx-push:
@echo "🚀 Building and pushing multi-architecture production Docker images with buildx..."
./docker-buildx.sh --push
.PHONY: docker-buildx-version
docker-buildx-version:
@if [ -z "$(VERSION)" ]; then \
echo "❌ Error: Please specify version, example: make docker-buildx-version VERSION=v1.0.0"; \
exit 1; \
fi
@echo "🏗️ Building multi-architecture production Docker images (version: $(VERSION))..."
./docker-buildx.sh --release $(VERSION)
.PHONY: docker-buildx-push-version
docker-buildx-push-version:
@if [ -z "$(VERSION)" ]; then \
echo "❌ Error: Please specify version, example: make docker-buildx-push-version VERSION=v1.0.0"; \
exit 1; \
fi
@echo "🚀 Building and pushing multi-architecture production Docker images (version: $(VERSION))..."
./docker-buildx.sh --release $(VERSION) --push
# Development/Source builds using direct buildx commands
.PHONY: docker-dev
docker-dev:
@echo "🏗️ Building multi-architecture development Docker images with buildx..."
@echo "💡 This builds from source code and is intended for local development and testing"
@echo "⚠️ Multi-arch images cannot be loaded locally, use docker-dev-push to push to registry"
$(DOCKER_CLI) buildx build \
--platform linux/amd64,linux/arm64 \
--file $(DOCKERFILE_SOURCE) \
--tag rustfs:source-latest \
--tag rustfs:dev-latest \
.
.PHONY: docker-dev-local
docker-dev-local:
@echo "🏗️ Building single-architecture development Docker image for local use..."
@echo "💡 This builds from source code for the current platform and loads locally"
$(DOCKER_CLI) buildx build \
--file $(DOCKERFILE_SOURCE) \
--tag rustfs:source-latest \
--tag rustfs:dev-latest \
--load \
.
.PHONY: docker-dev-push
docker-dev-push:
@if [ -z "$(REGISTRY)" ]; then \
echo "❌ Error: Please specify registry, example: make docker-dev-push REGISTRY=ghcr.io/username"; \
exit 1; \
fi
@echo "🚀 Building and pushing multi-architecture development Docker images..."
@echo "💡 Pushing to registry: $(REGISTRY)"
$(DOCKER_CLI) buildx build \
--platform linux/amd64,linux/arm64 \
--file $(DOCKERFILE_SOURCE) \
--tag $(REGISTRY)/rustfs:source-latest \
--tag $(REGISTRY)/rustfs:dev-latest \
--push \
.
endef
export HEADER
-include $(addsuffix /*.mak, $(shell find .config/make -type d))
# Local production builds using direct buildx (alternative to docker-buildx.sh)
.PHONY: docker-buildx-production-local
docker-buildx-production-local:
@echo "🏗️ Building single-architecture production Docker image locally..."
@echo "💡 Alternative to docker-buildx.sh for local testing"
$(DOCKER_CLI) buildx build \
--file $(DOCKERFILE_PRODUCTION) \
--tag rustfs:production-latest \
--tag rustfs:latest \
--load \
--build-arg RELEASE=latest \
.
# ========================================================================================
# Single Architecture Docker Builds (Traditional)
# ========================================================================================
.PHONY: docker-build-production
docker-build-production:
@echo "🏗️ Building single-architecture production Docker image..."
@echo "💡 Consider using 'make docker-buildx-production-local' for multi-arch support"
$(DOCKER_CLI) build -f $(DOCKERFILE_PRODUCTION) -t rustfs:latest .
.PHONY: docker-build-source
docker-build-source:
@echo "🏗️ Building single-architecture source Docker image..."
@echo "💡 Consider using 'make docker-dev-local' for multi-arch support"
DOCKER_BUILDKIT=1 $(DOCKER_CLI) build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
-f $(DOCKERFILE_SOURCE) -t rustfs:source .
# ========================================================================================
# Development Environment
# ========================================================================================
.PHONY: dev-env-start
dev-env-start:
@echo "🚀 Starting development environment..."
$(DOCKER_CLI) buildx build \
--file $(DOCKERFILE_SOURCE) \
--tag rustfs:dev \
--load \
.
$(DOCKER_CLI) stop $(CONTAINER_NAME) 2>/dev/null || true
$(DOCKER_CLI) rm $(CONTAINER_NAME) 2>/dev/null || true
$(DOCKER_CLI) run -d --name $(CONTAINER_NAME) \
-p 9010:9010 -p 9000:9000 \
-v $(shell pwd):/workspace \
-it rustfs:dev
.PHONY: dev-env-stop
dev-env-stop:
@echo "🛑 Stopping development environment..."
$(DOCKER_CLI) stop $(CONTAINER_NAME) 2>/dev/null || true
$(DOCKER_CLI) rm $(CONTAINER_NAME) 2>/dev/null || true
.PHONY: dev-env-restart
dev-env-restart: dev-env-stop dev-env-start
# ========================================================================================
# Build Utilities
# ========================================================================================
.PHONY: docker-inspect-multiarch
docker-inspect-multiarch:
@if [ -z "$(IMAGE)" ]; then \
echo "❌ Error: Please specify image, example: make docker-inspect-multiarch IMAGE=rustfs/rustfs:latest"; \
exit 1; \
fi
@echo "🔍 Inspecting multi-architecture image: $(IMAGE)"
docker buildx imagetools inspect $(IMAGE)
.PHONY: build-cross-all
build-cross-all:
@echo "🔧 Building all target architectures..."
@echo "💡 On macOS/Windows, use 'make docker-dev' for reliable multi-arch builds"
@echo "🔨 Generating protobuf code..."
cargo run --bin gproto || true
@echo "🔨 Building x86_64-unknown-linux-gnu..."
./build-rustfs.sh --platform x86_64-unknown-linux-gnu
@echo "🔨 Building aarch64-unknown-linux-gnu..."
./build-rustfs.sh --platform aarch64-unknown-linux-gnu
@echo "🔨 Building x86_64-unknown-linux-musl..."
./build-rustfs.sh --platform x86_64-unknown-linux-musl
@echo "🔨 Building aarch64-unknown-linux-musl..."
./build-rustfs.sh --platform aarch64-unknown-linux-musl
@echo "✅ All architectures built successfully!"
# ========================================================================================
# Help and Documentation
# ========================================================================================
.PHONY: help-build
help-build:
@echo "🔨 RustFS Build Help:"
@echo ""
@echo "🚀 Local Build (Recommended):"
@echo " make build # Build RustFS binary (includes console by default)"
@echo " make build-dev # Development mode build"
@echo " make build-musl # Build x86_64 musl version"
@echo " make build-gnu # Build x86_64 GNU version"
@echo " make build-musl-arm64 # Build aarch64 musl version"
@echo " make build-gnu-arm64 # Build aarch64 GNU version"
@echo ""
@echo "🐳 Docker Build:"
@echo " make build-docker # Build using Docker container"
@echo " make build-docker BUILD_OS=ubuntu22.04 # Specify build system"
@echo ""
@echo "🏗️ Cross-architecture Build:"
@echo " make build-cross-all # Build binaries for all architectures"
@echo ""
@echo "🔧 Direct usage of build-rustfs.sh script:"
@echo " ./build-rustfs.sh --help # View script help"
@echo " ./build-rustfs.sh --no-console # Build without console resources"
@echo " ./build-rustfs.sh --force-console-update # Force update console resources"
@echo " ./build-rustfs.sh --dev # Development mode build"
@echo " ./build-rustfs.sh --sign # Sign binary files"
@echo " ./build-rustfs.sh --platform x86_64-unknown-linux-gnu # Specify target platform"
@echo " ./build-rustfs.sh --skip-verification # Skip binary verification"
@echo ""
@echo "💡 build-rustfs.sh script provides more options, smart detection and binary verification"
.PHONY: help-docker
help-docker:
@echo "🐳 Docker Multi-architecture Build Help:"
@echo ""
@echo "🚀 Production Image Build (Recommended to use docker-buildx.sh):"
@echo " make docker-buildx # Build production multi-arch image (no push)"
@echo " make docker-buildx-push # Build and push production multi-arch image"
@echo " make docker-buildx-version VERSION=v1.0.0 # Build specific version"
@echo " make docker-buildx-push-version VERSION=v1.0.0 # Build and push specific version"
@echo ""
@echo "🔧 Development/Source Image Build (Local development testing):"
@echo " make docker-dev # Build dev multi-arch image (cannot load locally)"
@echo " make docker-dev-local # Build dev single-arch image (local load)"
@echo " make docker-dev-push REGISTRY=xxx # Build and push dev image"
@echo ""
@echo "🏗️ Local Production Image Build (Alternative):"
@echo " make docker-buildx-production-local # Build production single-arch image locally"
@echo ""
@echo "📦 Single-architecture Build (Traditional way):"
@echo " make docker-build-production # Build single-arch production image"
@echo " make docker-build-source # Build single-arch source image"
@echo ""
@echo "🚀 Development Environment Management:"
@echo " make dev-env-start # Start development container environment"
@echo " make dev-env-stop # Stop development container environment"
@echo " make dev-env-restart # Restart development container environment"
@echo ""
@echo "🔧 Auxiliary Tools:"
@echo " make build-cross-all # Build binaries for all architectures"
@echo " make docker-inspect-multiarch IMAGE=xxx # Check image architecture support"
@echo ""
@echo "📋 Environment Variables:"
@echo " REGISTRY Image registry address (required for push)"
@echo " DOCKERHUB_USERNAME Docker Hub username"
@echo " DOCKERHUB_TOKEN Docker Hub access token"
@echo " GITHUB_TOKEN GitHub access token"
@echo ""
@echo "💡 Suggestions:"
@echo " - Production use: Use docker-buildx* commands (based on precompiled binaries)"
@echo " - Local development: Use docker-dev* commands (build from source)"
@echo " - Development environment: Use dev-env-* commands to manage dev containers"
.PHONY: help
help:
@echo "🦀 RustFS Makefile Help:"
@echo ""
@echo "📋 Main Command Categories:"
@echo " make help-build # Show build-related help"
@echo " make help-docker # Show Docker-related help"
@echo ""
@echo "🔧 Code Quality:"
@echo " make fmt # Format code"
@echo " make clippy # Run clippy checks"
@echo " make test # Run tests"
@echo " make pre-commit # Run all pre-commit checks"
@echo ""
@echo "🚀 Quick Start:"
@echo " make build # Build RustFS binary"
@echo " make docker-dev-local # Build development Docker image (local)"
@echo " make dev-env-start # Start development environment"
@echo ""
@echo "💡 For more help use 'make help-build' or 'make help-docker'"
+111 -148
View File
@@ -1,6 +1,6 @@
[![RustFS](https://github.com/user-attachments/assets/1b5afcd6-a2c3-47ff-8bc3-ce882b0ddca7)](https://rustfs.com)
[![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
<p align="center">RustFS is a high-performance, distributed object storage system built in Rust.</p>
<p align="center">RustFS is a high-performance distributed object storage software built using Rust</p>
<p align="center">
<a href="https://github.com/rustfs/rustfs/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/rustfs/rustfs/actions/workflows/ci.yml/badge.svg" /></a>
@@ -11,12 +11,7 @@
</p>
<p align="center">
<a href="https://trendshift.io/repositories/14181" target="_blank"><img src="https://trendshift.io/api/badge/repositories/14181" alt="rustfs%2Frustfs | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
<a href="https://runacap.com/ross-index/q4-2025/" target="_blank" rel="noopener"><img style="width: 260px; height: 55px" src="https://runacap.com/wp-content/uploads/2026/01/ROSS_badge_white_Q4_2025.svg" alt="ROSS Index - Fastest Growing Open-Source Startups in Q4 2025 | Runa Capital" height="55" /></a>
</p>
<p align="center">
<a href="https://docs.rustfs.com/installation/">Getting Started</a>
<a href="https://docs.rustfs.com/introduction.html">Getting Started</a>
· <a href="https://docs.rustfs.com/">Docs</a>
· <a href="https://github.com/rustfs/rustfs/issues">Bug reports</a>
· <a href="https://github.com/rustfs/rustfs/discussions">Discussions</a>
@@ -24,6 +19,7 @@
<p align="center">
English | <a href="https://github.com/rustfs/rustfs/blob/main/README_ZH.md">简体中文</a> |
<!-- Keep these links. Translations will automatically update with the README. -->
<a href="https://readme-i18n.com/rustfs/rustfs?lang=de">Deutsch</a> |
<a href="https://readme-i18n.com/rustfs/rustfs?lang=es">Español</a> |
<a href="https://readme-i18n.com/rustfs/rustfs?lang=fr">français</a> |
@@ -33,186 +29,143 @@ English | <a href="https://github.com/rustfs/rustfs/blob/main/README_ZH.md">简
<a href="https://readme-i18n.com/rustfs/rustfs?lang=ru">Русский</a>
</p>
RustFS is a high-performance, distributed object storage system built in Rust—one of the most loved programming languages worldwide. RustFS combines the simplicity of MinIO with the memory safety and raw performance of Rust. It offers full S3 compatibility, is completely open-source, and is optimized for data lakes, AI, and big data workloads.
RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages
worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature,
support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in
comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation,
RustFS provides faster speed and safer distributed features for high-performance object storage.
Unlike other storage systems, RustFS is released under the permissible Apache 2.0 license, avoiding the restrictions of AGPL. With Rust as its foundation, RustFS delivers superior speed and secure distributed features for next-generation object storage.
> ⚠️ **RustFS is under rapid development. Do NOT use in production environments!**
## Feature & Status
## Features
- **High Performance**: Built with Rust to ensure maximum speed and resource efficiency.
- **Distributed Architecture**: Scalable and fault-tolerant design suitable for large-scale deployments.
- **S3 Compatibility**: Seamless integration with existing S3-compatible applications and tools.
- **Data Lake Support**: Optimized for high-throughput big data and AI workloads.
- **Open Source**: Licensed under Apache 2.0, encouraging unrestricted community contributions and commercial usage.
- **User-Friendly**: Designed with simplicity in mind for easy deployment and management.
- **High Performance**: Built with Rust, ensuring speed and efficiency.
- **Distributed Architecture**: Scalable and fault-tolerant design for large-scale deployments.
- **S3 Compatibility**: Seamless integration with existing S3-compatible applications.
- **Data Lake Support**: Optimized for big data and AI workloads.
- **Open Source**: Licensed under Apache 2.0, encouraging community contributions and transparency.
- **User-Friendly**: Designed with simplicity in mind, making it easy to deploy and manage.
| Feature | Status | Feature | Status |
| :---------------------- | :----------- | :----------------------- | :--------------- |
| **S3 Core Features** | ✅ Available | **Bitrot Protection** | ✅ Available |
| **Upload / Download** | ✅ Available | **Single Node Mode** | ✅ Available |
| **Versioning** | ✅ Available | **Bucket Replication** | ✅ Available |
| **Logging** | ✅ Available | **Lifecycle Management** | 🚧 Under Testing |
| **Event Notifications** | ✅ Available | **Distributed Mode** | 🚧 Under Testing |
| **K8s Helm Charts** | ✅ Available | **RustFS KMS** | 🚧 Under Testing |
## RustFS vs MinIO
## RustFS vs MinIO Performance
Stress test server parameters
**Stress Test Environment:**
| Type | Parameter | Remark |
| ------- | --------- | -------------------------------------------------------- |
| CPU | 2 Core | Intel Xeon (Sapphire Rapids) Platinum 8475B, 2.7/3.2 GHz |
| Memory | 4GB | |
| Network | 15Gbps | |
| Drive | 40GB x 4 | IOPS 3800 / Drive |
| Type | parameter | Remark |
|---------|-----------|----------------------------------------------------------|
| CPU | 2 Core | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz | |
| Memory | 4GB |   |
| Network | 15Gbp |   |
| Driver | 40GB x 4 | IOPS 3800 / Driver |
<https://github.com/user-attachments/assets/2e4979b5-260c-4f2c-ac12-c87fd558072a>
### RustFS vs Other Object Storage
### RustFS vs Other object storage
| Feature | RustFS | Other Object Storage |
| :--------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------- |
| **Console Experience** | **Powerful Console**<br>Comprehensive management interface. | **Basic / Limited Console**<br>Often overly simple or lacking critical features. |
| **Language & Safety** | **Rust-based**<br>Memory safety by design. | **Go or C-based**<br>Potential for memory GC pauses or leaks. |
| **Data Sovereignty** | **No Telemetry / Full Compliance**<br>Guards against unauthorized cross-border data egress. Compliant with GDPR (EU/UK), CCPA (US), and APPI (Japan). | **Potential Risk**<br>Possible legal exposure and unwanted data telemetry. |
| **Licensing** | **Permissive Apache 2.0**<br>Business-friendly, no "poison pill" clauses. | **Restrictive AGPL v3**<br>Risk of license traps and intellectual property pollution. |
| **Compatibility** | **100% S3 Compatible**<br>Works with any cloud provider or client, anywhere. | **Variable Compatibility**<br>May lack support for local cloud vendors or specific APIs. |
| **Edge & IoT** | **Strong Edge Support**<br>Ideal for secure, innovative edge devices. | **Weak Edge Support**<br>Often too heavy for edge gateways. |
| **Risk Profile** | **Enterprise Risk Mitigation**<br>Clear IP rights and safe for commercial use. | **Legal Risks**<br>Intellectual property ambiguity and usage restrictions. |
## Staying ahead
Star RustFS on GitHub and be instantly notified of new releases.
<img src="https://github.com/user-attachments/assets/7ee40bb4-3e46-4eac-b0d0-5fbeb85ff8f3" />
| RustFS | Other object storage |
|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
| Powerful Console | Simple and useless Console |
| Developed based on Rust language, memory is safer | Developed in Go or C, with potential issues like memory GC/leaks |
| Does not report logs to third-party countries | Reporting logs to other third countries may violate national security laws |
| Licensed under Apache, more business-friendly | AGPL V3 License and other License, polluted open source and License traps, infringement of intellectual property rights |
| Comprehensive S3 support, works with domestic and international cloud providers | Full support for S3, but no local cloud vendor support |
| Rust-based development, strong support for secure and innovative devices | Poor support for edge gateways and secure innovative devices |
| Stable commercial prices, free community support | High pricing, with costs up to $250,000 for 1PiB |
| No risk | Intellectual property risks and risks of prohibited uses |
## Quickstart
To get started with RustFS, follow these steps:
### 1. One-click Installation (Option 1)
1. **One-click installation script (Option 1)**
```bash
curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
```
```bash
curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
```
### 2\. Docker Quick Start (Option 2)
2. **Docker Quick Start (Option 2)**
The RustFS container runs as a non-root user `rustfs` (UID `10001`). If you run Docker with `-v` to mount a host directory, please ensure the host directory owner is set to `10001`, otherwise you will encounter permission denied errors.
```bash
# create data and logs directories
mkdir -p data logs
```bash
# Create data and logs directories
mkdir -p data logs
# using latest alpha version
docker run -d -p 9000:9000 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:alpha
# Change the owner of these directories
chown -R 10001:10001 data logs
# Specific version
docker run -d -p 9000:9000 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0.alpha.45
```
# Using latest version
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
For docker installation, you can also run the container with docker compose. With the `docker-compose.yml` file under
root directory, running the command:
# Using specific version
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-alpha.76
```
```
docker compose --profile observability up -d
```
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
**NOTE**: You should be better to have a look for `docker-compose.yaml` file. Because, several services contains in the
file. Grafan,prometheus,jaeger containers will be launched using docker compose file, which is helpful for rustfs
observability. If you want to start redis as well as nginx container, you can specify the corresponding profiles.
```bash
podman run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
```
3. **Build from Source (Option 3) - Advanced Users**
You can also use Docker Compose. Using the `docker-compose.yml` file in the root directory:
For developers who want to build RustFS Docker images from source with multi-architecture support:
```bash
docker compose --profile observability up -d
```
```bash
# Build multi-architecture images locally
./docker-buildx.sh --build-arg RELEASE=latest
Similarly, you can run the command with podman
# Build and push to registry
./docker-buildx.sh --push
```bash
podman compose --profile observability up -d
```
# Build specific version
./docker-buildx.sh --release v1.0.0 --push
**NOTE**: We recommend reviewing the `docker-compose.yaml` file before running. It defines several services including Grafana, Prometheus, and Jaeger, which are helpful for RustFS observability. If you wish to start Redis or Nginx containers, you can specify the corresponding profiles.
# Build for custom registry
./docker-buildx.sh --registry your-registry.com --namespace yourname --push
```
### 3\. Build from Source (Option 3) - Advanced Users
The `docker-buildx.sh` script supports:
- **Multi-architecture builds**: `linux/amd64`, `linux/arm64`
- **Automatic version detection**: Uses git tags or commit hashes
- **Registry flexibility**: Supports Docker Hub, GitHub Container Registry, etc.
- **Build optimization**: Includes caching and parallel builds
For developers who want to build RustFS Docker images from source with multi-architecture support:
You can also use Make targets for convenience:
```bash
# Build multi-architecture images locally
./docker-buildx.sh --build-arg RELEASE=latest
```bash
make docker-buildx # Build locally
make docker-buildx-push # Build and push
make docker-buildx-version VERSION=v1.0.0 # Build specific version
make help-docker # Show all Docker-related commands
```
# Build and push to registry
./docker-buildx.sh --push
4. **Build with helm chart(Option 4) - Cloud Native environment**
# Build specific version
./docker-buildx.sh --release v1.0.0 --push
Following the instructions on [helm chart README](./helm/README.md) to install RustFS on kubernetes cluster.
# Build for custom registry
./docker-buildx.sh --registry your-registry.com --namespace yourname --push
```
5. **Access the Console**: Open your web browser and navigate to `http://localhost:9000` to access the RustFS console,
default username and password is `rustfsadmin` .
6. **Create a Bucket**: Use the console to create a new bucket for your objects.
7. **Upload Objects**: You can upload files directly through the console or use S3-compatible APIs to interact with your
RustFS instance.
The `docker-buildx.sh` script supports:
- **Multi-architecture builds**: `linux/amd64`, `linux/arm64`
- **Automatic version detection**: Uses git tags or commit hashes
- **Registry flexibility**: Supports Docker Hub, GitHub Container Registry, etc.
- **Build optimization**: Includes caching and parallel builds
You can also use Make targets for convenience:
```bash
make docker-buildx # Build locally
make docker-buildx-push # Build and push
make docker-buildx-version VERSION=v1.0.0 # Build specific version
make help-docker # Show all Docker-related commands
```
> **Heads-up (macOS cross-compilation)**: macOS keeps the default `ulimit -n` at 256, so `cargo zigbuild` or `./build-rustfs.sh --platform ...` may fail with `ProcessFdQuotaExceeded` when targeting Linux. The build script attempts to raise the limit automatically, but if you still see the warning, run `ulimit -n 4096` (or higher) in your shell before building.
### 4\. Build with Helm Chart (Option 4) - Cloud Native
Follow the instructions in the [Helm Chart README](https://charts.rustfs.com/) to install RustFS on a Kubernetes cluster.
### 5\. Nix Flake (Option 5)
If you have [Nix with flakes enabled](https://nixos.wiki/wiki/Flakes#Enable_flakes):
```bash
# Run directly without installing
nix run github:rustfs/rustfs
# Build the binary
nix build github:rustfs/rustfs
./result/bin/rustfs --help
# Or from a local checkout
nix build
nix run
```
---
### Accessing RustFS
1. **Access the Console**: Open your web browser and navigate to `http://localhost:9001` to access the RustFS console.
- Default credentials: `rustfsadmin` / `rustfsadmin`
2. **Create a Bucket**: Use the console to create a new bucket for your objects.
3. **Upload Objects**: You can upload files directly through the console or use S3-compatible APIs/clients to interact with your RustFS instance.
**NOTE**: To access the RustFS instance via `https`, please refer to the [TLS Configuration Docs](https://docs.rustfs.com/integration/tls-configured.html).
**NOTE**: If you want to access RustFS instance with `https`, you can refer
to [TLS configuration docs](https://docs.rustfs.com/integration/tls-configured.html).
## Documentation
For detailed documentation, including configuration options, API references, and advanced usage, please visit our [Documentation](https://docs.rustfs.com).
For detailed documentation, including configuration options, API references, and advanced usage, please visit
our [Documentation](https://docs.rustfs.com).
## Getting Help
If you have any questions or need assistance:
If you have any questions or need assistance, you can:
- Check the [FAQ](https://github.com/rustfs/rustfs/discussions/categories/q-a) for common issues and solutions.
- Join our [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) to ask questions and share your experiences.
- Open an issue on our [GitHub Issues](https://github.com/rustfs/rustfs/issues) page for bug reports or feature requests.
- Join our [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) to ask questions and share your
experiences.
- Open an issue on our [GitHub Issues](https://github.com/rustfs/rustfs/issues) page for bug reports or feature
requests.
## Links
@@ -223,19 +176,28 @@ If you have any questions or need assistance:
## Contact
- **Bugs**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
- **Business**: [hello@rustfs.com](mailto:hello@rustfs.com)
- **Jobs**: [jobs@rustfs.com](mailto:jobs@rustfs.com)
- **Business**: <hello@rustfs.com>
- **Jobs**: <jobs@rustfs.com>
- **General Discussion**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
- **Contributing**: [CONTRIBUTING.md](CONTRIBUTING.md)
## Contributors
RustFS is a community-driven project, and we appreciate all contributions. Check out the [Contributors](https://github.com/rustfs/rustfs/graphs/contributors) page to see the amazing people who have helped make RustFS better.
RustFS is a community-driven project, and we appreciate all contributions. Check out
the [Contributors](https://github.com/rustfs/rustfs/graphs/contributors) page to see the amazing people who have helped
make RustFS better.
<a href="https://github.com/rustfs/rustfs/graphs/contributors">
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" alt="Contributors" />
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" alt="Contributors"/>
</a>
## Github Trending Top
🚀 RustFS is beloved by open-source enthusiasts and enterprise users worldwide, often appearing on the GitHub Trending
top charts.
<a href="https://trendshift.io/repositories/14181" target="_blank"><img src="https://raw.githubusercontent.com/rustfs/rustfs/refs/heads/main/docs/rustfs-trending.jpg" alt="rustfs%2Frustfs | Trendshift" /></a>
## Star History
[![Star History Chart](https://api.star-history.com/svg?repos=rustfs/rustfs&type=date&legend=top-left)](https://www.star-history.com/#rustfs/rustfs&type=date&legend=top-left)
@@ -245,3 +207,4 @@ RustFS is a community-driven project, and we appreciate all contributions. Check
[Apache 2.0](https://opensource.org/licenses/Apache-2.0)
**RustFS** is a trademark of RustFS, Inc. All other trademarks are the property of their respective owners.
+114 -150
View File
@@ -1,216 +1,180 @@
[![RustFS](https://github.com/user-attachments/assets/1b5afcd6-a2c3-47ff-8bc3-ce882b0ddca7)](https://rustfs.com.cn)
[![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
<p align="center">RustFS 是一个基于 Rust 构建的高性能分布式对象存储系统。</p>
<p align="center">RustFS 是一个使用 Rust 构建的高性能分布式对象存储软件</p >
<p align="center">
<a href="https://github.com/rustfs/rustfs/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/rustfs/rustfs/actions/workflows/ci.yml/badge.svg" /></a>
<a href="https://github.com/rustfs/rustfs/actions/workflows/docker.yml"><img alt="构建并推送 Docker 镜像" src="https://github.com/rustfs/rustfs/actions/workflows/docker.yml/badge.svg" /></a>
<img alt="GitHub 提交活跃度" src="https://img.shields.io/github/commit-activity/m/rustfs/rustfs"/>
<img alt="Github 最新提交" src="https://img.shields.io/github/last-commit/rustfs/rustfs"/>
<a href="https://github.com/rustfs/rustfs/actions/workflows/docker.yml"><img alt="Build and Push Docker Images" src="https://github.com/rustfs/rustfs/actions/workflows/docker.yml/badge.svg" /></a>
<img alt="GitHub commit activity" src="https://img.shields.io/github/commit-activity/m/rustfs/rustfs"/>
<img alt="Github Last Commit" src="https://img.shields.io/github/last-commit/rustfs/rustfs"/>
<a href="https://hellogithub.com/repository/rustfs/rustfs" target="_blank"><img src="https://abroad.hellogithub.com/v1/widgets/recommend.svg?rid=b95bcb72bdc340b68f16fdf6790b7d5b&claim_uid=MsbvjYeLDKAH457&theme=small" alt="FeaturedHelloGitHub" /></a>
</p>
</p >
<p align="center">
<a href="https://trendshift.io/repositories/14181" target="_blank"><img src="https://trendshift.io/api/badge/repositories/14181" alt="rustfs%2Frustfs | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
<a href="https://runacap.com/ross-index/q4-2025/" target="_blank" rel="noopener"><img style="width: 260px; height: 55px" src="https://runacap.com/wp-content/uploads/2026/01/ROSS_badge_white_Q4_2025.svg" alt="ROSS Index - Fastest Growing Open-Source Startups in Q4 2025 | Runa Capital" height="55" /></a>
</p>
<a href="https://docs.rustfs.com/zh/introduction.html">快速开始</a >
· <a href="https://docs.rustfs.com/zh/">文档</a >
· <a href="https://github.com/rustfs/rustfs/issues">问题报告</a >
· <a href="https://github.com/rustfs/rustfs/discussions">讨论</a >
</p >
<p align="center">
<a href="https://docs.rustfs.com/installation/">快速开始</a>
· <a href="https://docs.rustfs.com/">文档</a>
· <a href="https://github.com/rustfs/rustfs/issues">报告 Bug</a>
· <a href="https://github.com/rustfs/rustfs/discussions">社区讨论</a>
</p>
<a href="https://github.com/rustfs/rustfs/blob/main/README.md">English</a > | 简体中文
</p >
<p align="center">
<a href="https://github.com/rustfs/rustfs/blob/main/README.md">English</a> | 简体中文 |
<a href="https://readme-i18n.com/rustfs/rustfs?lang=de">Deutsch</a> |
<a href="https://readme-i18n.com/rustfs/rustfs?lang=es">Español</a> |
<a href="https://readme-i18n.com/rustfs/rustfs?lang=fr">français</a> |
<a href="https://readme-i18n.com/rustfs/rustfs?lang=ja">日本語</a> |
<a href="https://readme-i18n.com/rustfs/rustfs?lang=ko">한국어</a> |
<a href="https://readme-i18n.com/rustfs/rustfs?lang=pt">Portuguese</a> |
<a href="https://readme-i18n.com/rustfs/rustfs?lang=ru">Русский</a>
</p>
RustFS 是一个使用 Rust(全球最受欢迎的编程语言之一)构建的高性能分布式对象存储软件。与 MinIO 一样,它具有简单性、S3
兼容性、开源特性以及对数据湖、AI 和大数据的支持等一系列优势。此外,与其他存储系统相比,它采用 Apache
许可证构建,拥有更好、更用户友好的开源许可证。由于以 Rust 为基础,RustFS 为高性能对象存储提供了更快的速度和更安全的分布式功能。
RustFS 是一个基于 Rust 构建的高性能分布式对象存储系统。Rust 是全球最受开发者喜爱的编程语言之一,RustFS 完美结合了 MinIO 的简洁性与 Rust 的内存安全及高性能优势。它提供完整的 S3 兼容性,完全开源,并专为数据湖、人工智能(AI)和大数据负载进行了优化。
## 特性
与其他存储系统不同,RustFS 采用更宽松、商业友好的 Apache 2.0 许可证,避免了 AGPL 协议的限制。以 Rust 为基石,RustFS 为下一代对象存储提供了更快的速度和更安全的分布式特性
## 特征和功能状态
- **高性能**:基于 Rust 构建,确保极致的速度和资源效率。
- **高性能**:使用 Rust 构建,确保速度和效率
- **分布式架构**:可扩展且容错的设计,适用于大规模部署。
- **S3 兼容性**:与现有 S3 兼容应用和工具无缝集成。
- **数据湖支持**专为高吞吐量的大数据和 AI 工作负载优化。
- **完全开源**:采用 Apache 2.0 许可证,鼓励社区贡献和商业使用
- **简单易用**:设计简,易于部署和管理。
- **S3 兼容性**:与现有 S3 兼容应用程序无缝集成。
- **数据湖支持**针对大数据和 AI 工作负载进行了优化。
- **开源**:采用 Apache 2.0 许可证,鼓励社区贡献和透明度
- **用户友好**:设计简,易于部署和管理。
| 功能 | 状态 | 功能 | 状态 |
| :----------------- | :------ | :---------------------- | :-------- |
| **S3 核心功能** | ✅ 可用 | **Bitrot (防数据腐烂)** | ✅ 可用 |
| **上传 / 下载** | ✅ 可用 | **单机模式** | ✅ 可用 |
| **版本控制** | ✅ 可用 | **存储桶复制** | ✅ 可用 |
| **日志功能** | ✅ 可用 | **生命周期管理** | 🚧 测试中 |
| **事件通知** | ✅ 可用 | **分布式模式** | 🚧 测试中 |
| **K8s Helm Chart** | ✅ 可用 | **OPA (策略引擎)** | 🚧 测试中 |
## RustFS vs MinIO
## RustFS vs MinIO 性能对比
压力测试服务器参数
**压力测试环境参数:**
| 类型 | 参数 | 备注 |
| ---- | -------- | --------------------------------------------------------- |
| CPU | 2 核 | Intel Xeon (Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz |
| 内存 | 4GB |   |
| 网络 | 15Gbps |   |
| 硬盘 | 40GB x 4 | IOPS 3800 / Drive |
| 类型 | 参数 | 备注 |
|-----|----------|----------------------------------------------------------|
| CPU | 2 核心 | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz | |
| 内存 | 4GB | |
| 网络 | 15Gbp | |
| 驱动器 | 40GB x 4 | IOPS 3800 / 驱动器 |
<https://github.com/user-attachments/assets/2e4979b5-260c-4f2c-ac12-c87fd558072a>
### RustFS vs 其他对象存储
| 特性 | RustFS | 其他对象存储 |
| :------------- | :------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------- |
| **控制台体验** | **功能强大的控制台**<br>提供全面的管理界面。 | **基础/简陋的控制台**<br>通常功能过于简单或缺失关键特性。 |
| **语言与安全** | **基于 Rust 开发**<br>天生的内存安全 | **基于 Go 或 C 开发**<br>存在内存 GC 停顿或内存泄漏潜在风险。 |
| **数据主权** | **无遥测 / 完全合规**<br>防止未经授权的数据跨境传输。完全符合 GDPR (欧盟/英国)、CCPA (美国) 和 APPI (日本) 等法规。 | **潜在风险**<br>可能存在法律风险和隐蔽的数据遥测(Telemetry)。 |
| **开源协议** | **宽松的 Apache 2.0**<br>商业友好,无“毒丸”条款。 | **受限的 AGPL v3**<br>存在许可证陷阱知识产权污染的风险。 |
| **兼容性** | **100% S3 兼容**<br>适用于任何云提供商和客户端,随处运行。 | **兼容性不一**<br>虽然支持 S3,但可能缺乏对本地云厂商或特定 API 的支持。 |
| **边缘与 IoT** | **强大的边缘支持**<br>非常适合安全创新的边缘设备。 | **边缘支持较弱**<br>对于边缘网关来说通常过于沉重。 |
| **成本** | **稳定且免费**<br>免费社区支持,稳定的商业定价。 | **高昂成本**<br>1PiB 成本可能高达 250,000 美元。 |
| **风险控制** | **企业级风险规避**<br>清晰的知识产权,商业使用安全无忧。 | **法律风险**<br>知识产权归属模糊及使用限制风险。 |
## 保持领先
在 GitHub 上为 RustFS 点赞,即可第一时间收到新版本发布通知。
<img src="https://github.com/user-attachments/assets/7ee40bb4-3e46-4eac-b0d0-5fbeb85ff8f3" />
| RustFS | 其他对象存储 |
|--------------------------|-------------------------------------|
| 强大的控制台 | 简单且无用的控制台 |
| 基于 Rust 语言开发,内存安全 | 使用 Go 或 C 开发存在内存 GC/泄漏潜在问题 |
| 不向第三方国家报告日志 | 向其他第三方国家报告日志可能违反国家安全法律 |
| 采用 Apache 许可证,对商业友好 | AGPL V3 许可证等其他许可证,污染开源和许可证陷阱,侵犯知识产权 |
| 全面的 S3 支持,适用于国内外云提供商 | 完全支持 S3,但不支持本地云厂商 |
| 基于 Rust 开发,对安全创新设备有强大支持 | 对边缘网关和安全创新设备支持较差 |
| 稳定的商业价格,免费社区支持 | 高昂的定价,1PiB 成本高达 $250,000 |
| 无风险 | 知识产权风险和禁止使用的风险 |
## 快速开始
请按照以下步骤快速上手 RustFS
要开始使用 RustFS,请按照以下步骤操作
### 1. 一键安装脚本 (选项 1)
1. **一键脚本快速启动 (方案一)**
```bash
curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
```
```bash
curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
```
### 2\. Docker 快速启动 (选项 2)
2. **Docker 快速启动(方案二)**
RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用 Docker 的 `-v` 参数挂载宿主机目录,请务必确保宿主机目录的所有者已更改为 `1000`,否则会遇到权限拒绝错误。
```bash
docker run -d -p 9000:9000 -v /data:/data rustfs/rustfs
```
```bash
# 创建数据和日志目录
mkdir -p data logs
对于使用 Docker 安装来讲,你还可以使用 `docker compose` 来启动 rustfs 实例。在仓库的根目录下面有一个 `docker-compose.yml`
文件。运行如下命令即可:
# 更改这两个目录的所有者
chown -R 10001:10001 data logs
```
docker compose --profile observability up -d
```
# 使用最新版本运行
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
**注意**:在使用 `docker compose` 之前,你应该仔细阅读一下 `docker-compose.yaml`,因为该文件中包含多个服务,除了 rustfs
以外,还有 grafana、prometheus、jaeger 等,这些是为 rustfs 可观测性服务的,还有 redis 和 nginx。你想启动哪些容器,就需要用
`--profile` 参数指定相应的 profile。
# 使用指定版本运行
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0.alpha.68
```
3. **从源码构建(方案三)- 高级用户**
您也可以使用 Docker Compose。使用根目录下的 `docker-compose.yml` 文件
面向希望从源码构建支持多架构 Docker 镜像的开发者
```bash
docker compose --profile observability up -d
```
```bash
# 本地构建多架构镜像
./docker-buildx.sh --build-arg RELEASE=latest
**注意**: 我们建议您在运行前查看 `docker-compose.yaml` 文件。该文件定义了包括 Grafana、Prometheus 和 Jaeger 在内的多个服务,有助于 RustFS 的可观测性监控。如果您还想启动 Redis 或 Nginx 容器,可以指定相应的 profile。
# 构建并推送至镜像仓库
./docker-buildx.sh --push
### 3\. 源码编译 (选项 3) - 进阶用户
# 构建指定版本
./docker-buildx.sh --release v1.0.0 --push
适用于希望从源码构建支持多架构 RustFS Docker 镜像的开发者:
# 构建并推送到自定义镜像仓库
./docker-buildx.sh --registry your-registry.com --namespace yourname --push
```
```bash
# 在本地构建多架构镜像
./docker-buildx.sh --build-arg RELEASE=latest
`docker-buildx.sh` 脚本支持:
- **多架构构建**`linux/amd64`、`linux/arm64`
- **自动版本检测**:可使用 git 标签或提交哈希
- **仓库灵活性**:支持 Docker Hub、GitHub Container Registry 等
- **构建优化**:包含缓存和并行构建
# 构建并推送到仓库
./docker-buildx.sh --push
你也可以使用 Makefile 提供的目标命令以提升便捷性:
# 构建指定版本
./docker-buildx.sh --release v1.0.0 --push
```bash
make docker-buildx # 本地构建
make docker-buildx-push # 构建并推送
make docker-buildx-version VERSION=v1.0.0 # 构建指定版本
make help-docker # 显示全部 Docker 相关命令
```
# 构建并推送到自定义仓库
./docker-buildx.sh --registry your-registry.com --namespace yourname --push
```
4. **使用 Helm Chart 部署(方案四)- 云原生环境**
`docker-buildx.sh` 脚本支持:
- **多架构构建**: `linux/amd64`, `linux/arm64`
- **自动版本检测**: 使用 git tags 或 commit hash
- **灵活的仓库支持**: 支持 Docker Hub, GitHub Container Registry 等
- **构建优化**: 包含缓存和并行构建
按照 [helm chart 说明文档](./helm/README.md) 的指引,在 Kubernetes 集群中安装 RustFS。
为了方便起见,您也可以使用 Make 命令:
5. **访问控制台**:打开 Web 浏览器并导航到 `http://localhost:9000` 以访问 RustFS 控制台,默认的用户名和密码是
`rustfsadmin` 。
6. **创建存储桶**:使用控制台为您的对象创建新的存储桶。
7. **上传对象**:您可以直接通过控制台上传文件,或使用 S3 兼容的 API 与您的 RustFS 实例交互。
```bash
make docker-buildx # 本地构建
make docker-buildx-push # 构建并推送
make docker-buildx-version VERSION=v1.0.0 # 构建指定版本
make help-docker # 显示所有 Docker 相关命令
```
> **注意 (macOS 交叉编译)**: macOS 默认的 `ulimit -n` 限制为 256,因此在使用 `cargo zigbuild` 或 `./build-rustfs.sh --platform ...` 交叉编译 Linux 版本时,可能会因 `ProcessFdQuotaExceeded` 失败。构建脚本会尝试自动提高限制,但如果您仍然看到警告,请在构建前在终端运行 `ulimit -n 4096` (或更高)。
### 4\. 使用 Helm Chart 安装 (选项 4) - 云原生环境
请按照 [Helm Chart README](https://charts.rustfs.com) 上的说明在 Kubernetes 集群上安装 RustFS。
---
### 访问 RustFS
1. **访问控制台**: 打开浏览器并访问 `http://localhost:9000` 进入 RustFS 控制台。
- 默认账号/密码: `rustfsadmin` / `rustfsadmin`
2. **创建存储桶**: 使用控制台为您​​的对象创建一个新的存储桶 (Bucket)。
3. **上传对象**: 您可以直接通过控制台上传文件,或使用 S3 兼容的 API/客户端与您的 RustFS 实例进行交互。
**注意**: 如果您希望通过 `https` 访问 RustFS 实例,请参考 [TLS 配置文档](https://docs.rustfs.com/integration/tls-configured.html)。
**注意**:如果你想通过 `https` 来访问 RustFS 实例,请参考 [TLS 配置文档](https://docs.rustfs.com/zh/integration/tls-configured.html)
## 文档
有关详细文档,包括配置选项、API 参考和高级用法,请访问我们的 [官方文档](https://docs.rustfs.com)。
有关详细文档,包括配置选项、API 参考和高级用法,请访问我们的[文档](https://docs.rustfs.com)。
## 获取帮助
如果您有任何问题或需要帮助:
如果您有任何问题或需要帮助,您可以
- 查看 [FAQ](https://github.com/rustfs/rustfs/discussions/categories/q-a) 寻找常见问题和解决方案。
- 加入我们的 [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) 提问分享您的经验。
- 在我们的 [GitHub Issues](https://github.com/rustfs/rustfs/issues) 页面提交 Bug 报告或功能请求。
- 查看[常见问题解答](https://github.com/rustfs/rustfs/discussions/categories/q-a)以获取常见问题和解决方案。
- 加入我们的 [GitHub 讨论](https://github.com/rustfs/rustfs/discussions)提问分享您的经验。
- 在我们的 [GitHub Issues](https://github.com/rustfs/rustfs/issues) 页面上开启问题,报告错误或功能请求。
## 链接
- [官方文档](https://docs.rustfs.com) - 必读手册
- [更新日志](https://github.com/rustfs/rustfs/releases) - 版本变更记录
- [社区讨论](https://github.com/rustfs/rustfs/discussions) - 社区交流
- [文档](https://docs.rustfs.com) - 您应该阅读的手册
- [更新日志](https://docs.rustfs.com/changelog) - 我们破坏和修复的内容
- [GitHub 讨论](https://github.com/rustfs/rustfs/discussions) - 社区所在
## 联系方式
## 联系
- **Bug 反馈**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
- **商务合作**: [hello@rustfs.com](mailto:hello@rustfs.com)
- **工作机会**: [jobs@rustfs.com](mailto:jobs@rustfs.com)
- **一般讨论**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
- **贡献指南**: [CONTRIBUTING.md](https://www.google.com/search?q=CONTRIBUTING.md)
- **错误报告**[GitHub Issues](https://github.com/rustfs/rustfs/issues)
- **商务合作**<hello@rustfs.com>
- **招聘**<jobs@rustfs.com>
- **一般讨论**[GitHub 讨论](https://github.com/rustfs/rustfs/discussions)
- **贡献**[CONTRIBUTING.md](CONTRIBUTING.md)
## 贡献者
RustFS 是一个社区驱动的项目,我们感谢所有的贡献。查看 [贡献者](https://github.com/rustfs/rustfs/graphs/contributors) 页面,看看那些让 RustFS 变得更好的了不起的人们。
RustFS 是一个社区驱动的项目,我们感谢所有的贡献。查看[贡献者](https://github.com/rustfs/rustfs/graphs/contributors)页面,了解帮助
RustFS 变得更好的杰出人员。
<a href="https://github.com/rustfs/rustfs/graphs/contributors">
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" alt="Contributors" />
</a>
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" alt="贡献者"/>
</a >
## Star 历史
## Github 全球推荐榜
[![Star History Chart](https://api.star-history.com/svg?repos=rustfs/rustfs&type=date&legend=top-left)](https://www.star-history.com/#rustfs/rustfs&type=date&legend=top-left)
🚀 RustFS 受到了全世界开源爱好者和企业用户的喜欢,多次登顶 Github Trending 全球榜。
<a href="https://trendshift.io/repositories/14181" target="_blank"><img src="https://raw.githubusercontent.com/rustfs/rustfs/refs/heads/main/docs/rustfs-trending.jpg" alt="rustfs%2Frustfs | Trendshift" /></a>
## Star 历史图
[![Star 历史图](https://api.star-history.com/svg?repos=rustfs/rustfs&type=date&legend=top-left)](https://www.star-history.com/#rustfs/rustfs&type=date&legend=top-left)
## 许可证
+7 -29
View File
@@ -1,40 +1,18 @@
# Security Policy
## Security Philosophy
At RustFS, we take security seriously. We believe that **transparency leads to better security**. The more open our code is, the more eyes are on it, and the faster we can identify and resolve potential issues.
We highly value the contributions of the security community and welcome anyone to audit our code. Your efforts help us make RustFS safer for everyone.
## Supported Versions
To help us focus our security efforts, please refer to the table below to see which versions of RustFS are currently supported with security updates.
Use this section to tell people about which versions of your project are
currently being supported with security updates.
| Version | Supported |
| ------- | ------------------ |
| Latest | :white_check_mark: |
| < 1.0 | :x: |
| 1.x.x | :white_check_mark: |
## Reporting a Vulnerability
If you discover a security vulnerability in RustFS, we appreciate your help in disclosing it to us responsibly.
Use this section to tell people how to report a vulnerability.
**Please do not open a public GitHub issue for security vulnerabilities.** Publicly disclosing a vulnerability can put the entire community at risk before a fix is available.
### How to Report
1. https://github.com/rustfs/rustfs/security/advisories/new
2. Please email us directly at: **security@rustfs.com**
In your email, please include:
1. **Description**: A detailed description of the vulnerability.
2. **Steps to Reproduce**: Steps or a script to reproduce the issue.
3. **Impact**: The potential impact of the vulnerability.
### Our Response Process
1. **Acknowledgment**: We will acknowledge your email within 48 hours.
2. **Assessment**: We will investigate the issue and determine its severity.
3. **Fix & Disclosure**: We will work on a patch. Once the patch is released, we will publicly announce the vulnerability and acknowledge your contribution (unless you prefer to remain anonymous).
Thank you for helping keep RustFS and its users safe!
Tell them where to go, how often they can expect to get an update on a
reported vulnerability, what to expect if the vulnerability is accepted or
declined, etc.
+1 -7
View File
@@ -36,12 +36,6 @@ clen = "clen"
datas = "datas"
bre = "bre"
abd = "abd"
mak = "mak"
gae = "gae"
GAE = "GAE"
# s3-tests original test names (cannot be changed)
nonexisted = "nonexisted"
consts = "consts"
[files]
extend-exclude = []
extend-exclude = []
+3 -34
View File
@@ -1,4 +1,4 @@
#!/usr/bin/env bash
#!/bin/bash
# RustFS Binary Build Script
# This script compiles RustFS binaries for different platforms and architectures
@@ -31,7 +31,7 @@ detect_platform() {
echo "armv7-unknown-linux-gnueabihf"
;;
"loongarch64")
echo "loongarch64-unknown-linux-gnu"
echo "loongarch64-unknown-linux-musl"
;;
*)
echo "unknown-platform"
@@ -126,7 +126,6 @@ usage() {
echo " Supported platforms:"
echo " x86_64-unknown-linux-gnu"
echo " aarch64-unknown-linux-gnu"
echo " loongarch64-unknown-linux-gnu"
echo " armv7-unknown-linux-gnueabihf"
echo " x86_64-unknown-linux-musl"
echo " aarch64-unknown-linux-musl"
@@ -164,35 +163,6 @@ print_message() {
echo -e "${color}${message}${NC}"
}
# Prevent zig/ld from hitting macOS file descriptor defaults during linking
ensure_file_descriptor_limit() {
local required_limit=4096
local current_limit
current_limit=$(ulimit -Sn 2>/dev/null || echo "")
if [ -z "$current_limit" ] || [ "$current_limit" = "unlimited" ]; then
return
fi
if (( current_limit >= required_limit )); then
return
fi
local hard_limit target_limit
hard_limit=$(ulimit -Hn 2>/dev/null || echo "")
target_limit=$required_limit
if [ -n "$hard_limit" ] && [ "$hard_limit" != "unlimited" ] && (( hard_limit < required_limit )); then
target_limit=$hard_limit
fi
if ulimit -Sn "$target_limit" 2>/dev/null; then
print_message $YELLOW "🔧 Increased open file limit from $current_limit to $target_limit to avoid ProcessFdQuotaExceeded"
else
print_message $YELLOW "⚠️ Unable to raise ulimit -n automatically (current: $current_limit, needed: $required_limit). Please run 'ulimit -n $required_limit' manually before building."
fi
}
# Get version from git
get_version() {
if git describe --abbrev=0 --tags >/dev/null 2>&1; then
@@ -600,11 +570,10 @@ main() {
fi
fi
ensure_file_descriptor_limit
# Start build process
build_rustfs
}
# Run main function
main
@@ -1,52 +1,43 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-heal"
name = "rustfs-ahm"
version.workspace = true
edition.workspace = true
authors = ["RustFS Team"]
license.workspace = true
description = "RustFS erasure set and object healing"
description = "RustFS AHM (Automatic Health Management) Scanner"
repository.workspace = true
rust-version.workspace = true
homepage.workspace = true
documentation = "https://docs.rs/rustfs-heal/latest/rustfs_heal/"
keywords = ["RustFS", "heal", "erasure-coding", "Minio"]
documentation = "https://docs.rs/rustfs-ahm/latest/rustfs_ahm/"
keywords = ["RustFS", "AHM", "health-management", "scanner", "Minio"]
categories = ["web-programming", "development-tools", "filesystem"]
[dependencies]
rustfs-config = { workspace = true }
rustfs-ecstore = { workspace = true }
rustfs-common = { workspace = true }
rustfs-filemeta = { workspace = true }
rustfs-madmin = { workspace = true }
rustfs-utils = { workspace = true }
tokio = { workspace = true, features = ["full"] }
tokio-util = { workspace = true }
tracing = { workspace = true }
serde = { workspace = true, features = ["derive"] }
time = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde"] }
anyhow = { workspace = true }
async-trait = { workspace = true }
futures = { workspace = true }
s3s = { workspace = true }
chrono = { workspace = true }
rand = { workspace = true }
reqwest = { workspace = true }
tempfile = { workspace = true }
walkdir = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true }
serial_test = { workspace = true }
tracing-subscriber = { workspace = true }
tempfile = { workspace = true }
walkdir = { workspace = true }
http = { workspace = true }
heed = { workspace = true }
@@ -14,9 +14,9 @@
use thiserror::Error;
/// Custom error type for heal operations
/// Custom error type for AHM operations
/// This enum defines various error variants that can occur during
/// the execution of heal-related tasks, such as I/O errors, storage errors,
/// the execution of AHM-related tasks, such as I/O errors, storage errors,
/// configuration errors, and specific errors related to healing operations.
#[derive(Debug, Error)]
pub enum Error {
@@ -41,6 +41,13 @@ pub enum Error {
#[error(transparent)]
Anyhow(#[from] anyhow::Error),
// Scanner
#[error("Scanner error: {0}")]
Scanner(String),
#[error("Metrics error: {0}")]
Metrics(String),
#[error("Serialization error: {0}")]
Serialization(String),
@@ -53,6 +60,7 @@ pub enum Error {
#[error("Invalid checkpoint: {0}")]
InvalidCheckpoint(String),
// Heal
#[error("Heal task not found: {task_id}")]
TaskNotFound { task_id: String },
@@ -81,7 +89,9 @@ pub enum Error {
ProgressTrackingFailed { message: String },
}
/// A specialized Result type for heal operations
/// A specialized Result type for AHM operations
///This type is a convenient alias for results returned by functions in the AHM crate,
/// using the custom Error type defined above.
pub type Result<T, E = Error> = std::result::Result<T, E>;
impl Error {
@@ -90,12 +90,7 @@ impl HealChannelProcessor {
/// Process start request
async fn process_start_request(&self, request: HealChannelRequest) -> Result<()> {
info!(
"Processing heal start request: {} for bucket: {}/{}",
request.id,
request.bucket,
request.object_prefix.as_deref().unwrap_or("")
);
info!("Processing heal start request: {} for bucket: {}", request.id, request.bucket);
// Convert channel request to heal request
let heal_request = self.convert_to_heal_request(request.clone())?;
@@ -183,7 +178,7 @@ impl HealChannelProcessor {
HealType::Object {
bucket: request.bucket.clone(),
object: prefix.clone(),
version_id: request.object_version_id.clone(),
version_id: None,
}
} else {
HealType::Bucket {
@@ -329,14 +324,6 @@ mod tests {
async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> crate::Result<Vec<String>> {
Ok(vec![])
}
async fn list_objects_for_heal_page(
&self,
_bucket: &str,
_prefix: &str,
_continuation_token: Option<&str>,
) -> crate::Result<(Vec<String>, Option<String>, bool)> {
Ok((vec![], None, false))
}
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> crate::Result<rustfs_ecstore::disk::DiskStore> {
Err(crate::Error::other("Not implemented in mock"))
}
@@ -366,7 +353,6 @@ mod tests {
id: "test-id".to_string(),
bucket: "test-bucket".to_string(),
object_prefix: None,
object_version_id: None,
disk: None,
priority: HealChannelPriority::Normal,
scan_mode: None,
@@ -395,7 +381,6 @@ mod tests {
id: "test-id".to_string(),
bucket: "test-bucket".to_string(),
object_prefix: Some("test-object".to_string()),
object_version_id: None,
disk: None,
priority: HealChannelPriority::High,
scan_mode: Some(HealScanMode::Deep),
@@ -427,7 +412,6 @@ mod tests {
id: "test-id".to_string(),
bucket: "test-bucket".to_string(),
object_prefix: None,
object_version_id: None,
disk: Some("pool_0_set_1".to_string()),
priority: HealChannelPriority::Critical,
scan_mode: None,
@@ -456,7 +440,6 @@ mod tests {
id: "test-id".to_string(),
bucket: "test-bucket".to_string(),
object_prefix: None,
object_version_id: None,
disk: Some("invalid-disk-id".to_string()),
priority: HealChannelPriority::Normal,
scan_mode: None,
@@ -492,7 +475,6 @@ mod tests {
id: "test-id".to_string(),
bucket: "test-bucket".to_string(),
object_prefix: None,
object_version_id: None,
disk: None,
priority: channel_priority,
scan_mode: None,
@@ -521,7 +503,6 @@ mod tests {
id: "test-id".to_string(),
bucket: "test-bucket".to_string(),
object_prefix: None,
object_version_id: None,
disk: None,
priority: HealChannelPriority::Normal,
scan_mode: None,
@@ -551,7 +532,6 @@ mod tests {
id: "test-id".to_string(),
bucket: "test-bucket".to_string(),
object_prefix: Some("".to_string()), // Empty prefix should be treated as bucket heal
object_version_id: None,
disk: None,
priority: HealChannelPriority::Normal,
scan_mode: None,
@@ -49,9 +49,8 @@ impl ErasureSetHealer {
}
/// execute erasure set heal with resume
#[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))]
pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> {
info!("Starting erasure set heal");
info!("Starting erasure set heal for {} buckets on set disk {}", buckets.len(), set_disk_id);
// 1. generate or get task id
let task_id = self.get_or_create_task_id(set_disk_id).await?;
@@ -232,7 +231,6 @@ impl ErasureSetHealer {
/// heal single bucket with resume
#[allow(clippy::too_many_arguments)]
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, _skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
async fn heal_bucket_with_resume(
&self,
bucket: &str,
@@ -245,7 +243,7 @@ impl ErasureSetHealer {
resume_manager: &ResumeManager,
checkpoint_manager: &CheckpointManager,
) -> Result<()> {
info!(target: "rustfs:heal:heal_bucket_with_resume" ,"Starting heal for bucket from object index {}", current_object_index);
info!(target: "rustfs:ahm:heal_bucket_with_resume" ,"Starting heal for bucket: {} from object index {}", bucket, current_object_index);
// 1. get bucket info
let _bucket_info = match self.storage.get_bucket_info(bucket).await? {
@@ -256,114 +254,82 @@ impl ErasureSetHealer {
}
};
// 2. process objects with pagination to avoid loading all objects into memory
let mut continuation_token: Option<String> = None;
let mut global_obj_idx = 0usize;
// 2. get objects to heal
let objects = self.storage.list_objects_for_heal(bucket, "").await?;
loop {
// Get one page of objects
let (objects, next_token, is_truncated) = self
.storage
.list_objects_for_heal_page(bucket, "", continuation_token.as_deref())
// 3. continue from checkpoint
for (obj_idx, object) in objects.iter().enumerate().skip(*current_object_index) {
// check if already processed
if checkpoint_manager.get_checkpoint().await.processed_objects.contains(object) {
continue;
}
// update current object
resume_manager
.set_current_item(Some(bucket.to_string()), Some(object.clone()))
.await?;
// Process objects in this page
for object in objects {
// Skip objects before the checkpoint
if global_obj_idx < *current_object_index {
global_obj_idx += 1;
// Check if object still exists before attempting heal
let object_exists = match self.storage.object_exists(bucket, object).await {
Ok(exists) => exists,
Err(e) => {
warn!("Failed to check existence of {}/{}: {}, skipping", bucket, object, e);
*current_object_index = obj_idx + 1;
continue;
}
};
// check if already processed
if checkpoint_manager.get_checkpoint().await.processed_objects.contains(&object) {
global_obj_idx += 1;
continue;
}
if !object_exists {
info!(
target: "rustfs:ahm:heal_bucket_with_resume" ,"Object {}/{} no longer exists, skipping heal (likely deleted intentionally)",
bucket, object
);
checkpoint_manager.add_processed_object(object.clone()).await?;
*successful_objects += 1; // Treat as successful - object is gone as intended
*current_object_index = obj_idx + 1;
continue;
}
// update current object
resume_manager
.set_current_item(Some(bucket.to_string()), Some(object.clone()))
.await?;
// heal object
let heal_opts = HealOpts {
scan_mode: HealScanMode::Normal,
remove: true,
recreate: true, // Keep recreate enabled for legitimate heal scenarios
..Default::default()
};
// Check if object still exists before attempting heal
let object_exists = match self.storage.object_exists(bucket, &object).await {
Ok(exists) => exists,
Err(e) => {
warn!("Failed to check existence of {}/{}: {}, marking as failed", bucket, object, e);
*failed_objects += 1;
checkpoint_manager.add_failed_object(object.clone()).await?;
global_obj_idx += 1;
*current_object_index = global_obj_idx;
continue;
}
};
if !object_exists {
info!(
target: "rustfs:heal:heal_bucket_with_resume" ,"Object {}/{} no longer exists, skipping heal (likely deleted intentionally)",
bucket, object
);
match self.storage.heal_object(bucket, object, None, &heal_opts).await {
Ok((_result, None)) => {
*successful_objects += 1;
checkpoint_manager.add_processed_object(object.clone()).await?;
*successful_objects += 1; // Treat as successful - object is gone as intended
global_obj_idx += 1;
*current_object_index = global_obj_idx;
continue;
info!("Successfully healed object {}/{}", bucket, object);
}
// heal object
let heal_opts = HealOpts {
scan_mode: HealScanMode::Normal,
remove: true,
recreate: true, // Keep recreate enabled for legitimate heal scenarios
..Default::default()
};
match self.storage.heal_object(bucket, &object, None, &heal_opts).await {
Ok((_result, None)) => {
*successful_objects += 1;
checkpoint_manager.add_processed_object(object.clone()).await?;
info!("Successfully healed object {}/{}", bucket, object);
}
Ok((_, Some(err))) => {
*failed_objects += 1;
checkpoint_manager.add_failed_object(object.clone()).await?;
warn!("Failed to heal object {}/{}: {}", bucket, object, err);
}
Err(err) => {
*failed_objects += 1;
checkpoint_manager.add_failed_object(object.clone()).await?;
warn!("Error healing object {}/{}: {}", bucket, object, err);
}
Ok((_, Some(err))) => {
*failed_objects += 1;
checkpoint_manager.add_failed_object(object.clone()).await?;
warn!("Failed to heal object {}/{}: {}", bucket, object, err);
}
*processed_objects += 1;
global_obj_idx += 1;
*current_object_index = global_obj_idx;
// check cancel status
if self.cancel_token.is_cancelled() {
info!("Heal task cancelled during object processing");
return Err(Error::TaskCancelled);
}
// save checkpoint periodically
if global_obj_idx.is_multiple_of(100) {
checkpoint_manager
.update_position(bucket_index, *current_object_index)
.await?;
Err(err) => {
*failed_objects += 1;
checkpoint_manager.add_failed_object(object.clone()).await?;
warn!("Error healing object {}/{}: {}", bucket, object, err);
}
}
// Check if there are more pages
if !is_truncated {
break;
*processed_objects += 1;
*current_object_index = obj_idx + 1;
// check cancel status
if self.cancel_token.is_cancelled() {
info!("Heal task cancelled during object processing");
return Err(Error::TaskCancelled);
}
continuation_token = next_token;
if continuation_token.is_none() {
warn!("List is truncated but no continuation token provided for {}", bucket);
break;
// save checkpoint periodically
if obj_idx % 100 == 0 {
checkpoint_manager
.update_position(bucket_index, *current_object_index)
.await?;
}
}
@@ -397,7 +363,7 @@ impl ErasureSetHealer {
let _permit = semaphore
.acquire()
.await
.map_err(|e| Error::other(format!("Failed to acquire semaphore for bucket heal: {e}")))?;
.map_err(|e| Error::other(format!("Failed to acquire semaphore for bucket heal: {}", e)))?;
if cancel_token.is_cancelled() {
return Err(Error::TaskCancelled);
@@ -429,12 +395,16 @@ impl ErasureSetHealer {
}
};
// 2. process objects with pagination to avoid loading all objects into memory
let mut continuation_token: Option<String> = None;
let mut total_scanned = 0u64;
let mut total_success = 0u64;
let mut total_failed = 0u64;
// 2. get objects to heal
let objects = storage.list_objects_for_heal(bucket, "").await?;
// 3. update progress
{
let mut p = progress.write().await;
p.objects_scanned += objects.len() as u64;
}
// 4. heal objects concurrently
let heal_opts = HealOpts {
scan_mode: HealScanMode::Normal,
remove: true, // remove corrupted data
@@ -442,65 +412,27 @@ impl ErasureSetHealer {
..Default::default()
};
loop {
// Get one page of objects
let (objects, next_token, is_truncated) = storage
.list_objects_for_heal_page(bucket, "", continuation_token.as_deref())
.await?;
let object_results = Self::heal_objects_concurrently(storage, bucket, &objects, &heal_opts, progress).await;
let page_count = objects.len() as u64;
total_scanned += page_count;
// 5. count results
let (success_count, failure_count) = object_results
.into_iter()
.fold((0, 0), |(success, failure), result| match result {
Ok(_) => (success + 1, failure),
Err(_) => (success, failure + 1),
});
// 3. update progress
{
let mut p = progress.write().await;
p.objects_scanned = total_scanned;
}
// 4. heal objects concurrently for this page
let object_results = Self::heal_objects_concurrently(storage, bucket, &objects, &heal_opts, progress).await;
// 5. count results for this page
let (success_count, failure_count) =
object_results
.into_iter()
.fold((0, 0), |(success, failure), result| match result {
Ok(_) => (success + 1, failure),
Err(_) => (success, failure + 1),
});
total_success += success_count;
total_failed += failure_count;
// 6. update progress
{
let mut p = progress.write().await;
p.objects_healed = total_success;
p.objects_failed = total_failed;
p.set_current_object(Some(format!("processing bucket: {bucket} (page)")));
}
// Check if there are more pages
if !is_truncated {
break;
}
continuation_token = next_token;
if continuation_token.is_none() {
warn!("List is truncated but no continuation token provided for {}", bucket);
break;
}
}
// 7. final progress update
// 6. update progress
{
let mut p = progress.write().await;
p.objects_healed += success_count;
p.objects_failed += failure_count;
p.set_current_object(Some(format!("completed bucket: {bucket}")));
}
info!(
"Completed heal for bucket {}: {} success, {} failures (total scanned: {})",
bucket, total_success, total_failed, total_scanned
"Completed heal for bucket {}: {} success, {} failures",
bucket, success_count, failure_count
);
Ok(())
@@ -529,7 +461,7 @@ impl ErasureSetHealer {
let _permit = semaphore
.acquire()
.await
.map_err(|e| Error::other(format!("Failed to acquire semaphore for object heal: {e}")))?;
.map_err(|e| Error::other(format!("Failed to acquire semaphore for object heal: {}", e)))?;
match storage.heal_object(&bucket, &object, None, &heal_opts).await {
Ok((_result, None)) => {
+440
View File
@@ -0,0 +1,440 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::heal::{
progress::{HealProgress, HealStatistics},
storage::HealStorageAPI,
task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType},
};
use crate::{Error, Result};
use rustfs_ecstore::disk::DiskAPI;
use rustfs_ecstore::disk::error::DiskError;
use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP;
use std::{
collections::{HashMap, VecDeque},
sync::Arc,
time::{Duration, SystemTime},
};
use tokio::{
sync::{Mutex, RwLock},
time::interval,
};
use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn};
/// Heal config
#[derive(Debug, Clone)]
pub struct HealConfig {
/// Whether to enable auto heal
pub enable_auto_heal: bool,
/// Heal interval
pub heal_interval: Duration,
/// Maximum concurrent heal tasks
pub max_concurrent_heals: usize,
/// Task timeout
pub task_timeout: Duration,
/// Queue size
pub queue_size: usize,
}
impl Default for HealConfig {
fn default() -> Self {
Self {
enable_auto_heal: true,
heal_interval: Duration::from_secs(10), // 10 seconds
max_concurrent_heals: 4,
task_timeout: Duration::from_secs(300), // 5 minutes
queue_size: 1000,
}
}
}
/// Heal state
#[derive(Debug, Default)]
pub struct HealState {
/// Whether running
pub is_running: bool,
/// Current heal cycle
pub current_cycle: u64,
/// Last heal time
pub last_heal_time: Option<SystemTime>,
/// Total healed objects
pub total_healed_objects: u64,
/// Total heal failures
pub total_heal_failures: u64,
/// Current active heal tasks
pub active_heal_count: usize,
}
/// Heal manager
pub struct HealManager {
/// Heal config
config: Arc<RwLock<HealConfig>>,
/// Heal state
state: Arc<RwLock<HealState>>,
/// Active heal tasks
active_heals: Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
/// Heal queue
heal_queue: Arc<Mutex<VecDeque<HealRequest>>>,
/// Storage layer interface
storage: Arc<dyn HealStorageAPI>,
/// Cancel token
cancel_token: CancellationToken,
/// Statistics
statistics: Arc<RwLock<HealStatistics>>,
}
impl HealManager {
/// Create new HealManager
pub fn new(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
let config = config.unwrap_or_default();
Self {
config: Arc::new(RwLock::new(config)),
state: Arc::new(RwLock::new(HealState::default())),
active_heals: Arc::new(Mutex::new(HashMap::new())),
heal_queue: Arc::new(Mutex::new(VecDeque::new())),
storage,
cancel_token: CancellationToken::new(),
statistics: Arc::new(RwLock::new(HealStatistics::new())),
}
}
/// Start HealManager
pub async fn start(&self) -> Result<()> {
let mut state = self.state.write().await;
if state.is_running {
warn!("HealManager is already running");
return Ok(());
}
state.is_running = true;
drop(state);
info!("Starting HealManager");
// start scheduler
self.start_scheduler().await?;
// start auto disk scanner
self.start_auto_disk_scanner().await?;
info!("HealManager started successfully");
Ok(())
}
/// Stop HealManager
pub async fn stop(&self) -> Result<()> {
info!("Stopping HealManager");
// cancel all tasks
self.cancel_token.cancel();
// wait for all tasks to complete
let mut active_heals = self.active_heals.lock().await;
for task in active_heals.values() {
if let Err(e) = task.cancel().await {
warn!("Failed to cancel task {}: {}", task.id, e);
}
}
active_heals.clear();
// update state
let mut state = self.state.write().await;
state.is_running = false;
info!("HealManager stopped successfully");
Ok(())
}
/// Submit heal request
pub async fn submit_heal_request(&self, request: HealRequest) -> Result<String> {
let config = self.config.read().await;
let mut queue = self.heal_queue.lock().await;
if queue.len() >= config.queue_size {
return Err(Error::ConfigurationError {
message: "Heal queue is full".to_string(),
});
}
let request_id = request.id.clone();
queue.push_back(request);
drop(queue);
info!("Submitted heal request: {}", request_id);
Ok(request_id)
}
/// Get task status
pub async fn get_task_status(&self, task_id: &str) -> Result<HealTaskStatus> {
let active_heals = self.active_heals.lock().await;
if let Some(task) = active_heals.get(task_id) {
Ok(task.get_status().await)
} else {
Err(Error::TaskNotFound {
task_id: task_id.to_string(),
})
}
}
/// Get task progress
pub async fn get_active_tasks_count(&self) -> usize {
self.active_heals.lock().await.len()
}
pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
let active_heals = self.active_heals.lock().await;
if let Some(task) = active_heals.get(task_id) {
Ok(task.get_progress().await)
} else {
Err(Error::TaskNotFound {
task_id: task_id.to_string(),
})
}
}
/// Cancel task
pub async fn cancel_task(&self, task_id: &str) -> Result<()> {
let mut active_heals = self.active_heals.lock().await;
if let Some(task) = active_heals.get(task_id) {
task.cancel().await?;
active_heals.remove(task_id);
info!("Cancelled heal task: {}", task_id);
Ok(())
} else {
Err(Error::TaskNotFound {
task_id: task_id.to_string(),
})
}
}
/// Get statistics
pub async fn get_statistics(&self) -> HealStatistics {
self.statistics.read().await.clone()
}
/// Get active task count
pub async fn get_active_task_count(&self) -> usize {
let active_heals = self.active_heals.lock().await;
active_heals.len()
}
/// Get queue length
pub async fn get_queue_length(&self) -> usize {
let queue = self.heal_queue.lock().await;
queue.len()
}
/// Start scheduler
async fn start_scheduler(&self) -> Result<()> {
let config = self.config.clone();
let heal_queue = self.heal_queue.clone();
let active_heals = self.active_heals.clone();
let cancel_token = self.cancel_token.clone();
let statistics = self.statistics.clone();
let storage = self.storage.clone();
tokio::spawn(async move {
let mut interval = interval(config.read().await.heal_interval);
loop {
tokio::select! {
_ = cancel_token.cancelled() => {
info!("Heal scheduler received shutdown signal");
break;
}
_ = interval.tick() => {
Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage).await;
}
}
}
});
Ok(())
}
/// Start background task to auto scan local disks and enqueue erasure set heal requests
async fn start_auto_disk_scanner(&self) -> Result<()> {
let config = self.config.clone();
let heal_queue = self.heal_queue.clone();
let active_heals = self.active_heals.clone();
let cancel_token = self.cancel_token.clone();
let storage = self.storage.clone();
tokio::spawn(async move {
let mut interval = interval(config.read().await.heal_interval);
loop {
tokio::select! {
_ = cancel_token.cancelled() => {
info!("Auto disk scanner received shutdown signal");
break;
}
_ = interval.tick() => {
// Build list of endpoints that need healing
let mut endpoints = Vec::new();
for (_, disk_opt) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
if let Some(disk) = disk_opt {
// detect unformatted disk via get_disk_id()
if let Err(err) = disk.get_disk_id().await {
if err == DiskError::UnformattedDisk {
endpoints.push(disk.endpoint());
continue;
}
}
}
}
if endpoints.is_empty() {
continue;
}
// Get bucket list for erasure set healing
let buckets = match storage.list_buckets().await {
Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::<Vec<String>>(),
Err(e) => {
error!("Failed to get bucket list for auto healing: {}", e);
continue;
}
};
// Create erasure set heal requests for each endpoint
for ep in endpoints {
let Some(set_disk_id) =
crate::heal::utils::format_set_disk_id_from_i32(ep.pool_idx, ep.set_idx)
else {
warn!("Skipping endpoint {} without valid pool/set index", ep);
continue;
};
// skip if already queued or healing
// Use consistent lock order: queue first, then active_heals to avoid deadlock
let mut skip = false;
{
let queue = heal_queue.lock().await;
if queue.iter().any(|req| {
matches!(
&req.heal_type,
crate::heal::task::HealType::ErasureSet { set_disk_id: queued_id, .. }
if queued_id == &set_disk_id
)
}) {
skip = true;
}
}
if !skip {
let active = active_heals.lock().await;
if active.values().any(|task| {
matches!(
&task.heal_type,
crate::heal::task::HealType::ErasureSet { set_disk_id: active_id, .. }
if active_id == &set_disk_id
)
}) {
skip = true;
}
}
if skip {
continue;
}
// enqueue erasure set heal request for this disk
let req = HealRequest::new(
HealType::ErasureSet {
buckets: buckets.clone(),
set_disk_id: set_disk_id.clone(),
},
HealOptions::default(),
HealPriority::Normal,
);
let mut queue = heal_queue.lock().await;
queue.push_back(req);
info!("Enqueued auto erasure set heal for endpoint: {} (set_disk_id: {})", ep, set_disk_id);
}
}
}
}
});
Ok(())
}
/// Process heal queue
async fn process_heal_queue(
heal_queue: &Arc<Mutex<VecDeque<HealRequest>>>,
active_heals: &Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
config: &Arc<RwLock<HealConfig>>,
statistics: &Arc<RwLock<HealStatistics>>,
storage: &Arc<dyn HealStorageAPI>,
) {
let config = config.read().await;
let mut active_heals_guard = active_heals.lock().await;
// check if new heal tasks can be started
if active_heals_guard.len() >= config.max_concurrent_heals {
return;
}
let mut queue = heal_queue.lock().await;
if let Some(request) = queue.pop_front() {
let task = Arc::new(HealTask::from_request(request, storage.clone()));
let task_id = task.id.clone();
active_heals_guard.insert(task_id.clone(), task.clone());
drop(active_heals_guard);
let active_heals_clone = active_heals.clone();
let statistics_clone = statistics.clone();
// start heal task
tokio::spawn(async move {
info!("Starting heal task: {}", task_id);
let result = task.execute().await;
match result {
Ok(_) => {
info!("Heal task completed successfully: {}", task_id);
}
Err(e) => {
error!("Heal task failed: {} - {}", task_id, e);
}
}
let mut active_heals_guard = active_heals_clone.lock().await;
if let Some(completed_task) = active_heals_guard.remove(&task_id) {
// update statistics
let mut stats = statistics_clone.write().await;
match completed_task.get_status().await {
HealTaskStatus::Completed => {
stats.update_task_completion(true);
}
_ => {
stats.update_task_completion(false);
}
}
stats.update_running_tasks(active_heals_guard.len() as u64);
}
});
// update statistics
let mut stats = statistics.write().await;
stats.total_tasks += 1;
}
}
}
impl std::fmt::Debug for HealManager {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HealManager")
.field("config", &"<config>")
.field("state", &"<state>")
.field("active_heals_count", &"<active_heals>")
.field("queue_length", &"<queue>")
.finish()
}
}
@@ -13,7 +13,6 @@
// limitations under the License.
use crate::{Error, Result};
use rustfs_ecstore::disk::error::DiskError;
use rustfs_ecstore::disk::{BUCKET_META_PREFIX, DiskAPI, DiskStore, RUSTFS_META_BUCKET};
use serde::{Deserialize, Serialize};
use std::path::Path;
@@ -31,7 +30,7 @@ const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json";
/// Helper function to convert Path to &str, returning an error if conversion fails
fn path_to_str(path: &Path) -> Result<&str> {
path.to_str()
.ok_or_else(|| Error::other(format!("Invalid UTF-8 path: {path:?}")))
.ok_or_else(|| Error::other(format!("Invalid UTF-8 path: {:?}", path)))
}
/// resume state
@@ -181,9 +180,7 @@ impl ResumeManager {
};
// save initial state
if let Err(e) = manager.save_state().await {
warn!("Failed to save initial resume state: {}", e);
}
manager.save_state().await?;
Ok(manager)
}
@@ -300,15 +297,12 @@ impl ResumeManager {
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", state.task_id, RESUME_STATE_FILE));
let path_str = path_to_str(&file_path)?;
if let Err(e) = self.disk.write_all(RUSTFS_META_BUCKET, path_str, state_data.into()).await {
if matches!(e, DiskError::UnformattedDisk) {
warn!("Cannot save resume state: unformatted disk");
return Ok(());
}
return Err(Error::TaskExecutionFailed {
self.disk
.write_all(RUSTFS_META_BUCKET, path_str, state_data.into())
.await
.map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to save resume state: {e}"),
});
}
})?;
debug!("Saved resume state for task: {}", state.task_id);
Ok(())
@@ -401,9 +395,7 @@ impl CheckpointManager {
};
// save initial checkpoint
if let Err(e) = manager.save_checkpoint().await {
warn!("Failed to save initial checkpoint: {}", e);
}
manager.save_checkpoint().await?;
Ok(manager)
}
@@ -549,10 +541,10 @@ impl ResumeUtils {
for entry in entries {
if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) {
// Extract task ID from filename: {task_id}_ahm_resume_state.json
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}"))
&& !task_id.is_empty()
{
task_ids.push(task_id.to_string());
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}")) {
if !task_id.is_empty() {
task_ids.push(task_id.to_string());
}
}
}
}
@@ -107,21 +107,9 @@ pub trait HealStorageAPI: Send + Sync {
/// Heal format using ecstore
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)>;
/// List objects for healing (returns all objects, may use significant memory for large buckets)
///
/// WARNING: This method loads all objects into memory at once. For buckets with many objects,
/// consider using `list_objects_for_heal_page` instead to process objects in pages.
/// List objects for healing
async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result<Vec<String>>;
/// List objects for healing with pagination (returns one page and continuation token)
/// Returns (objects, next_continuation_token, is_truncated)
async fn list_objects_for_heal_page(
&self,
bucket: &str,
prefix: &str,
continuation_token: Option<&str>,
) -> Result<(Vec<String>, Option<String>, bool)>;
/// Get disk for resume functionality
async fn get_disk_for_resume(&self, set_disk_id: &str) -> Result<DiskStore>;
}
@@ -192,7 +180,8 @@ impl HealStorageAPI for ECStoreHealStorage {
MAX_READ_BYTES, bucket, object
);
return Err(Error::other(format!(
"Object too large: {n_read} bytes (max: {MAX_READ_BYTES} bytes) for {bucket}/{object}"
"Object too large: {} bytes (max: {} bytes) for {}/{}",
n_read, MAX_READ_BYTES, bucket, object
)));
}
}
@@ -412,13 +401,13 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.get_object_info(bucket, object, &Default::default()).await {
Ok(_) => Ok(true), // Object exists
Err(e) => {
// Map ObjectNotFound to false, other errors must be propagated!
// Map ObjectNotFound to false, other errors to false as well for safety
if matches!(e, rustfs_ecstore::error::StorageError::ObjectNotFound(_, _)) {
debug!("Object not found: {}/{}", bucket, object);
Ok(false)
} else {
error!("Error checking object existence {}/{}: {}", bucket, object, e);
Err(Error::other(e))
debug!("Error checking object existence {}/{}: {}", bucket, object, e);
Ok(false) // Treat errors as non-existence to be safe
}
}
}
@@ -505,67 +494,24 @@ impl HealStorageAPI for ECStoreHealStorage {
async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result<Vec<String>> {
debug!("Listing objects for heal: {}/{}", bucket, prefix);
warn!(
"list_objects_for_heal loads all objects into memory. For large buckets, consider using list_objects_for_heal_page instead."
);
let mut all_objects = Vec::new();
let mut continuation_token: Option<String> = None;
loop {
let (page_objects, next_token, is_truncated) = self
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref())
.await?;
all_objects.extend(page_objects);
if !is_truncated {
break;
}
continuation_token = next_token;
if continuation_token.is_none() {
warn!("List is truncated but no continuation token provided for {}/{}", bucket, prefix);
break;
}
}
info!("Found {} objects for heal in {}/{}", all_objects.len(), bucket, prefix);
Ok(all_objects)
}
async fn list_objects_for_heal_page(
&self,
bucket: &str,
prefix: &str,
continuation_token: Option<&str>,
) -> Result<(Vec<String>, Option<String>, bool)> {
debug!("Listing objects for heal (page): {}/{}", bucket, prefix);
const MAX_KEYS: i32 = 1000;
let continuation_token_opt = continuation_token.map(|s| s.to_string());
// Use list_objects_v2 to get objects with pagination
let list_info = match self
// Use list_objects_v2 to get objects
match self
.ecstore
.clone()
.list_objects_v2(bucket, prefix, continuation_token_opt, None, MAX_KEYS, false, None, false)
.list_objects_v2(bucket, prefix, None, None, 1000, false, None)
.await
{
Ok(info) => info,
Ok(list_info) => {
let objects: Vec<String> = list_info.objects.into_iter().map(|obj| obj.name).collect();
info!("Found {} objects for heal in {}/{}", objects.len(), bucket, prefix);
Ok(objects)
}
Err(e) => {
error!("Failed to list objects for heal: {}/{} - {}", bucket, prefix, e);
return Err(Error::other(e));
Err(Error::other(e))
}
};
// Collect objects from this page
let page_objects: Vec<String> = list_info.objects.into_iter().map(|obj| obj.name).collect();
let page_count = page_objects.len();
debug!("Listed {} objects (page) for heal in {}/{}", page_count, bucket, prefix);
Ok((page_objects, list_info.next_continuation_token, list_info.is_truncated))
}
}
async fn get_disk_for_resume(&self, set_disk_id: &str) -> Result<DiskStore> {
@@ -51,7 +51,7 @@ pub enum HealType {
}
/// Heal priority
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub enum HealPriority {
/// Low priority
Low = 0,
@@ -272,7 +272,6 @@ impl HealTask {
}
}
#[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))]
pub async fn execute(&self) -> Result<()> {
// update status and timestamps atomically to avoid race conditions
let now = SystemTime::now();
@@ -286,7 +285,7 @@ impl HealTask {
*task_start_instant = Some(start_instant);
}
info!("Task started");
info!("Starting heal task: {} with type: {:?}", self.id, self.heal_type);
let result = match &self.heal_type {
HealType::Object {
@@ -316,7 +315,7 @@ impl HealTask {
Ok(_) => {
let mut status = self.status.write().await;
*status = HealTaskStatus::Completed;
info!("Task completed successfully");
info!("Heal task completed successfully: {}", self.id);
}
Err(Error::TaskCancelled) => {
let mut status = self.status.write().await;
@@ -355,9 +354,8 @@ impl HealTask {
}
// specific heal implementation method
#[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))]
async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!("Starting object heal workflow");
info!("Healing object: {}/{}", bucket, object);
// update progress
{
@@ -367,7 +365,7 @@ impl HealTask {
}
// Step 1: Check if object exists and get metadata
warn!("Step 1: Checking object existence and metadata");
info!("Step 1: Checking object existence and metadata");
self.check_control_flags().await?;
let object_exists = self.await_with_control(self.storage.object_exists(bucket, object)).await?;
if !object_exists {
@@ -426,7 +424,7 @@ impl HealTask {
// If heal failed and remove_corrupted is enabled, delete the corrupted object
if self.options.remove_corrupted {
info!("Removing corrupted object: {}/{}", bucket, object);
warn!("Removing corrupted object: {}/{}", bucket, object);
if !self.options.dry_run {
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
info!("Successfully deleted corrupted object: {}/{}", bucket, object);
@@ -449,9 +447,11 @@ impl HealTask {
info!("Step 3: Verifying heal result");
let object_size = result.object_size as u64;
info!(
object_size = object_size,
drives_healed = result.after.drives.len(),
"Heal completed successfully"
"Heal completed successfully: {}/{} ({} bytes, {} drives healed)",
bucket,
object,
object_size,
result.after.drives.len()
);
{
@@ -481,7 +481,7 @@ impl HealTask {
// If heal failed and remove_corrupted is enabled, delete the corrupted object
if self.options.remove_corrupted {
info!("Removing corrupted object: {}/{}", bucket, object);
warn!("Removing corrupted object: {}/{}", bucket, object);
if !self.options.dry_run {
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
info!("Successfully deleted corrupted object: {}/{}", bucket, object);
@@ -14,36 +14,38 @@
mod error;
pub mod heal;
pub mod scanner;
pub use error::{Error, Result};
pub use heal::{HealManager, HealOptions, HealPriority, HealRequest, HealType, channel::HealChannelProcessor};
pub use scanner::Scanner;
use std::sync::{Arc, OnceLock};
use tokio_util::sync::CancellationToken;
use tracing::{error, info};
// Global cancellation token for heal and related services
// Global cancellation token for AHM services (scanner and other background tasks)
static GLOBAL_AHM_SERVICES_CANCEL_TOKEN: OnceLock<CancellationToken> = OnceLock::new();
/// Initialize the global heal services cancellation token
/// Initialize the global AHM services cancellation token
pub fn init_ahm_services_cancel_token(cancel_token: CancellationToken) -> Result<()> {
GLOBAL_AHM_SERVICES_CANCEL_TOKEN
.set(cancel_token)
.map_err(|_| Error::Config("Heal services cancel token already initialized".to_string()))
.map_err(|_| Error::Config("AHM services cancel token already initialized".to_string()))
}
/// Get the global heal services cancellation token
/// Get the global AHM services cancellation token
pub fn get_ahm_services_cancel_token() -> Option<&'static CancellationToken> {
GLOBAL_AHM_SERVICES_CANCEL_TOKEN.get()
}
/// Create and initialize the global heal services cancellation token
/// Create and initialize the global AHM services cancellation token
pub fn create_ahm_services_cancel_token() -> CancellationToken {
let cancel_token = CancellationToken::new();
init_ahm_services_cancel_token(cancel_token.clone()).expect("Heal services cancel token already initialized");
init_ahm_services_cancel_token(cancel_token.clone()).expect("AHM services cancel token already initialized");
cancel_token
}
/// Shutdown all heal services gracefully
/// Shutdown all AHM services gracefully
pub fn shutdown_ahm_services() {
if let Some(cancel_token) = GLOBAL_AHM_SERVICES_CANCEL_TOKEN.get() {
cancel_token.cancel();
+326
View File
@@ -0,0 +1,326 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::scanner::node_scanner::ScanProgress;
use crate::{Error, Result};
use serde::{Deserialize, Serialize};
use std::{
path::{Path, PathBuf},
time::{Duration, SystemTime},
};
use tokio::sync::RwLock;
use tracing::{debug, error, info, warn};
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct CheckpointData {
pub version: u32,
pub timestamp: SystemTime,
pub progress: ScanProgress,
pub node_id: String,
pub checksum: u64,
}
impl CheckpointData {
pub fn new(progress: ScanProgress, node_id: String) -> Self {
let mut checkpoint = Self {
version: 1,
timestamp: SystemTime::now(),
progress,
node_id,
checksum: 0,
};
checkpoint.checksum = checkpoint.calculate_checksum();
checkpoint
}
fn calculate_checksum(&self) -> u64 {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
self.version.hash(&mut hasher);
self.node_id.hash(&mut hasher);
self.progress.current_cycle.hash(&mut hasher);
self.progress.current_disk_index.hash(&mut hasher);
if let Some(ref bucket) = self.progress.current_bucket {
bucket.hash(&mut hasher);
}
if let Some(ref key) = self.progress.last_scan_key {
key.hash(&mut hasher);
}
hasher.finish()
}
pub fn verify_integrity(&self) -> bool {
let calculated_checksum = self.calculate_checksum();
self.checksum == calculated_checksum
}
}
pub struct CheckpointManager {
checkpoint_file: PathBuf,
backup_file: PathBuf,
temp_file: PathBuf,
save_interval: Duration,
last_save: RwLock<SystemTime>,
node_id: String,
}
impl CheckpointManager {
pub fn new(node_id: &str, data_dir: &Path) -> Self {
if !data_dir.exists() {
if let Err(e) = std::fs::create_dir_all(data_dir) {
error!("create data dir failed {:?}: {}", data_dir, e);
}
}
let checkpoint_file = data_dir.join(format!("scanner_checkpoint_{node_id}.json"));
let backup_file = data_dir.join(format!("scanner_checkpoint_{node_id}.backup"));
let temp_file = data_dir.join(format!("scanner_checkpoint_{node_id}.tmp"));
Self {
checkpoint_file,
backup_file,
temp_file,
save_interval: Duration::from_secs(30), // 30s
last_save: RwLock::new(SystemTime::UNIX_EPOCH),
node_id: node_id.to_string(),
}
}
pub async fn save_checkpoint(&self, progress: &ScanProgress) -> Result<()> {
let now = SystemTime::now();
let last_save = *self.last_save.read().await;
if now.duration_since(last_save).unwrap_or(Duration::ZERO) < self.save_interval {
return Ok(());
}
let checkpoint_data = CheckpointData::new(progress.clone(), self.node_id.clone());
let json_data = serde_json::to_string_pretty(&checkpoint_data)
.map_err(|e| Error::Serialization(format!("serialize checkpoint failed: {e}")))?;
tokio::fs::write(&self.temp_file, json_data)
.await
.map_err(|e| Error::IO(format!("write temp checkpoint file failed: {e}")))?;
if self.checkpoint_file.exists() {
tokio::fs::copy(&self.checkpoint_file, &self.backup_file)
.await
.map_err(|e| Error::IO(format!("backup checkpoint file failed: {e}")))?;
}
tokio::fs::rename(&self.temp_file, &self.checkpoint_file)
.await
.map_err(|e| Error::IO(format!("replace checkpoint file failed: {e}")))?;
*self.last_save.write().await = now;
debug!(
"save checkpoint to {:?}, cycle: {}, disk index: {}",
self.checkpoint_file, checkpoint_data.progress.current_cycle, checkpoint_data.progress.current_disk_index
);
Ok(())
}
pub async fn load_checkpoint(&self) -> Result<Option<ScanProgress>> {
// first try main checkpoint file
match self.load_checkpoint_from_file(&self.checkpoint_file).await {
Ok(checkpoint) => {
info!(
"restore scan progress from main checkpoint file: cycle={}, disk index={}, last scan key={:?}",
checkpoint.current_cycle, checkpoint.current_disk_index, checkpoint.last_scan_key
);
Ok(Some(checkpoint))
}
Err(e) => {
warn!("main checkpoint file is corrupted or not exists: {}", e);
// try backup file
match self.load_checkpoint_from_file(&self.backup_file).await {
Ok(checkpoint) => {
warn!(
"restore scan progress from backup file: cycle={}, disk index={}",
checkpoint.current_cycle, checkpoint.current_disk_index
);
// copy backup file to main checkpoint file
if let Err(copy_err) = tokio::fs::copy(&self.backup_file, &self.checkpoint_file).await {
warn!("restore main checkpoint file failed: {}", copy_err);
}
Ok(Some(checkpoint))
}
Err(backup_e) => {
warn!("backup file is corrupted or not exists: {}", backup_e);
info!("cannot restore scan progress, will start fresh scan");
Ok(None)
}
}
}
}
}
/// load checkpoint from file
async fn load_checkpoint_from_file(&self, file_path: &Path) -> Result<ScanProgress> {
if !file_path.exists() {
return Err(Error::NotFound(format!("checkpoint file not exists: {file_path:?}")));
}
// read file content
let content = tokio::fs::read_to_string(file_path)
.await
.map_err(|e| Error::IO(format!("read checkpoint file failed: {e}")))?;
// deserialize
let checkpoint_data: CheckpointData =
serde_json::from_str(&content).map_err(|e| Error::Serialization(format!("deserialize checkpoint failed: {e}")))?;
// validate checkpoint data
self.validate_checkpoint(&checkpoint_data)?;
Ok(checkpoint_data.progress)
}
/// validate checkpoint data
fn validate_checkpoint(&self, checkpoint: &CheckpointData) -> Result<()> {
// validate data integrity
if !checkpoint.verify_integrity() {
return Err(Error::InvalidCheckpoint(
"checkpoint data verification failed, may be corrupted".to_string(),
));
}
// validate node id match
if checkpoint.node_id != self.node_id {
return Err(Error::InvalidCheckpoint(format!(
"checkpoint node id not match: expected {}, actual {}",
self.node_id, checkpoint.node_id
)));
}
let now = SystemTime::now();
let checkpoint_age = now.duration_since(checkpoint.timestamp).unwrap_or(Duration::MAX);
// checkpoint is too old (more than 24 hours), may be data expired
if checkpoint_age > Duration::from_secs(24 * 3600) {
return Err(Error::InvalidCheckpoint(format!("checkpoint data is too old: {checkpoint_age:?}")));
}
// validate version compatibility
if checkpoint.version > 1 {
return Err(Error::InvalidCheckpoint(format!(
"unsupported checkpoint version: {}",
checkpoint.version
)));
}
Ok(())
}
/// clean checkpoint file
///
/// called when scanner stops or resets
pub async fn cleanup_checkpoint(&self) -> Result<()> {
// delete main file
if self.checkpoint_file.exists() {
tokio::fs::remove_file(&self.checkpoint_file)
.await
.map_err(|e| Error::IO(format!("delete main checkpoint file failed: {e}")))?;
}
// delete backup file
if self.backup_file.exists() {
tokio::fs::remove_file(&self.backup_file)
.await
.map_err(|e| Error::IO(format!("delete backup checkpoint file failed: {e}")))?;
}
// delete temp file
if self.temp_file.exists() {
tokio::fs::remove_file(&self.temp_file)
.await
.map_err(|e| Error::IO(format!("delete temp checkpoint file failed: {e}")))?;
}
info!("cleaned up all checkpoint files");
Ok(())
}
/// get checkpoint file info
pub async fn get_checkpoint_info(&self) -> Result<Option<CheckpointInfo>> {
if !self.checkpoint_file.exists() {
return Ok(None);
}
let metadata = tokio::fs::metadata(&self.checkpoint_file)
.await
.map_err(|e| Error::IO(format!("get checkpoint file metadata failed: {e}")))?;
let content = tokio::fs::read_to_string(&self.checkpoint_file)
.await
.map_err(|e| Error::IO(format!("read checkpoint file failed: {e}")))?;
let checkpoint_data: CheckpointData =
serde_json::from_str(&content).map_err(|e| Error::Serialization(format!("deserialize checkpoint failed: {e}")))?;
Ok(Some(CheckpointInfo {
file_size: metadata.len(),
last_modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
checkpoint_timestamp: checkpoint_data.timestamp,
current_cycle: checkpoint_data.progress.current_cycle,
current_disk_index: checkpoint_data.progress.current_disk_index,
completed_disks_count: checkpoint_data.progress.completed_disks.len(),
is_valid: checkpoint_data.verify_integrity(),
}))
}
/// force save checkpoint (ignore time interval limit)
pub async fn force_save_checkpoint(&self, progress: &ScanProgress) -> Result<()> {
// temporarily reset last save time, force save
*self.last_save.write().await = SystemTime::UNIX_EPOCH;
self.save_checkpoint(progress).await
}
/// set save interval
pub async fn set_save_interval(&mut self, interval: Duration) {
self.save_interval = interval;
info!("checkpoint save interval set to: {:?}", interval);
}
}
/// checkpoint info
#[derive(Debug, Clone)]
pub struct CheckpointInfo {
/// file size
pub file_size: u64,
/// file last modified time
pub last_modified: SystemTime,
/// checkpoint creation time
pub checkpoint_timestamp: SystemTime,
/// current scan cycle
pub current_cycle: u64,
/// current disk index
pub current_disk_index: usize,
/// completed disks count
pub completed_disks_count: usize,
/// checkpoint is valid
pub is_valid: bool,
}
File diff suppressed because it is too large Load Diff
+305
View File
@@ -0,0 +1,305 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
sync::atomic::{AtomicU64, Ordering},
time::{Duration, SystemTime},
};
use tracing::info;
/// Scanner metrics
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ScannerMetrics {
/// Total objects scanned since server start
pub objects_scanned: u64,
/// Total object versions scanned since server start
pub versions_scanned: u64,
/// Total directories scanned since server start
pub directories_scanned: u64,
/// Total bucket scans started since server start
pub bucket_scans_started: u64,
/// Total bucket scans finished since server start
pub bucket_scans_finished: u64,
/// Total objects with health issues found
pub objects_with_issues: u64,
/// Total heal tasks queued
pub heal_tasks_queued: u64,
/// Total heal tasks completed
pub heal_tasks_completed: u64,
/// Total heal tasks failed
pub heal_tasks_failed: u64,
/// Total healthy objects found
pub healthy_objects: u64,
/// Total corrupted objects found
pub corrupted_objects: u64,
/// Last scan activity time
pub last_activity: Option<SystemTime>,
/// Current scan cycle
pub current_cycle: u64,
/// Total scan cycles completed
pub total_cycles: u64,
/// Current scan duration
pub current_scan_duration: Option<Duration>,
/// Average scan duration
pub avg_scan_duration: Duration,
/// Objects scanned per second
pub objects_per_second: f64,
/// Buckets scanned per second
pub buckets_per_second: f64,
/// Storage metrics by bucket
pub bucket_metrics: HashMap<String, BucketMetrics>,
/// Disk metrics
pub disk_metrics: HashMap<String, DiskMetrics>,
}
/// Bucket-specific metrics
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct BucketMetrics {
/// Bucket name
pub bucket: String,
/// Total objects in bucket
pub total_objects: u64,
/// Total size of objects in bucket (bytes)
pub total_size: u64,
/// Objects with health issues
pub objects_with_issues: u64,
/// Last scan time
pub last_scan_time: Option<SystemTime>,
/// Scan duration
pub scan_duration: Option<Duration>,
/// Heal tasks queued for this bucket
pub heal_tasks_queued: u64,
/// Heal tasks completed for this bucket
pub heal_tasks_completed: u64,
/// Heal tasks failed for this bucket
pub heal_tasks_failed: u64,
}
/// Disk-specific metrics
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DiskMetrics {
/// Disk path
pub disk_path: String,
/// Total disk space (bytes)
pub total_space: u64,
/// Used disk space (bytes)
pub used_space: u64,
/// Free disk space (bytes)
pub free_space: u64,
/// Objects scanned on this disk
pub objects_scanned: u64,
/// Objects with issues on this disk
pub objects_with_issues: u64,
/// Last scan time
pub last_scan_time: Option<SystemTime>,
/// Whether disk is online
pub is_online: bool,
/// Whether disk is being scanned
pub is_scanning: bool,
}
/// Thread-safe metrics collector
pub struct MetricsCollector {
/// Atomic counters for real-time metrics
objects_scanned: AtomicU64,
versions_scanned: AtomicU64,
directories_scanned: AtomicU64,
bucket_scans_started: AtomicU64,
bucket_scans_finished: AtomicU64,
objects_with_issues: AtomicU64,
heal_tasks_queued: AtomicU64,
heal_tasks_completed: AtomicU64,
heal_tasks_failed: AtomicU64,
current_cycle: AtomicU64,
total_cycles: AtomicU64,
healthy_objects: AtomicU64,
corrupted_objects: AtomicU64,
}
impl MetricsCollector {
/// Create a new metrics collector
pub fn new() -> Self {
Self {
objects_scanned: AtomicU64::new(0),
versions_scanned: AtomicU64::new(0),
directories_scanned: AtomicU64::new(0),
bucket_scans_started: AtomicU64::new(0),
bucket_scans_finished: AtomicU64::new(0),
objects_with_issues: AtomicU64::new(0),
heal_tasks_queued: AtomicU64::new(0),
heal_tasks_completed: AtomicU64::new(0),
heal_tasks_failed: AtomicU64::new(0),
current_cycle: AtomicU64::new(0),
total_cycles: AtomicU64::new(0),
healthy_objects: AtomicU64::new(0),
corrupted_objects: AtomicU64::new(0),
}
}
/// Increment objects scanned count
pub fn increment_objects_scanned(&self, count: u64) {
self.objects_scanned.fetch_add(count, Ordering::Relaxed);
}
/// Increment versions scanned count
pub fn increment_versions_scanned(&self, count: u64) {
self.versions_scanned.fetch_add(count, Ordering::Relaxed);
}
/// Increment directories scanned count
pub fn increment_directories_scanned(&self, count: u64) {
self.directories_scanned.fetch_add(count, Ordering::Relaxed);
}
/// Increment bucket scans started count
pub fn increment_bucket_scans_started(&self, count: u64) {
self.bucket_scans_started.fetch_add(count, Ordering::Relaxed);
}
/// Increment bucket scans finished count
pub fn increment_bucket_scans_finished(&self, count: u64) {
self.bucket_scans_finished.fetch_add(count, Ordering::Relaxed);
}
/// Increment objects with issues count
pub fn increment_objects_with_issues(&self, count: u64) {
self.objects_with_issues.fetch_add(count, Ordering::Relaxed);
}
/// Increment heal tasks queued count
pub fn increment_heal_tasks_queued(&self, count: u64) {
self.heal_tasks_queued.fetch_add(count, Ordering::Relaxed);
}
/// Increment heal tasks completed count
pub fn increment_heal_tasks_completed(&self, count: u64) {
self.heal_tasks_completed.fetch_add(count, Ordering::Relaxed);
}
/// Increment heal tasks failed count
pub fn increment_heal_tasks_failed(&self, count: u64) {
self.heal_tasks_failed.fetch_add(count, Ordering::Relaxed);
}
/// Set current cycle
pub fn set_current_cycle(&self, cycle: u64) {
self.current_cycle.store(cycle, Ordering::Relaxed);
}
/// Increment total cycles
pub fn increment_total_cycles(&self) {
self.total_cycles.fetch_add(1, Ordering::Relaxed);
}
/// Increment healthy objects count
pub fn increment_healthy_objects(&self) {
self.healthy_objects.fetch_add(1, Ordering::Relaxed);
}
/// Increment corrupted objects count
pub fn increment_corrupted_objects(&self) {
self.corrupted_objects.fetch_add(1, Ordering::Relaxed);
}
/// Get current metrics snapshot
pub fn get_metrics(&self) -> ScannerMetrics {
ScannerMetrics {
objects_scanned: self.objects_scanned.load(Ordering::Relaxed),
versions_scanned: self.versions_scanned.load(Ordering::Relaxed),
directories_scanned: self.directories_scanned.load(Ordering::Relaxed),
bucket_scans_started: self.bucket_scans_started.load(Ordering::Relaxed),
bucket_scans_finished: self.bucket_scans_finished.load(Ordering::Relaxed),
objects_with_issues: self.objects_with_issues.load(Ordering::Relaxed),
heal_tasks_queued: self.heal_tasks_queued.load(Ordering::Relaxed),
heal_tasks_completed: self.heal_tasks_completed.load(Ordering::Relaxed),
heal_tasks_failed: self.heal_tasks_failed.load(Ordering::Relaxed),
healthy_objects: self.healthy_objects.load(Ordering::Relaxed),
corrupted_objects: self.corrupted_objects.load(Ordering::Relaxed),
last_activity: Some(SystemTime::now()),
current_cycle: self.current_cycle.load(Ordering::Relaxed),
total_cycles: self.total_cycles.load(Ordering::Relaxed),
current_scan_duration: None, // Will be set by scanner
avg_scan_duration: Duration::ZERO, // Will be calculated
objects_per_second: 0.0, // Will be calculated
buckets_per_second: 0.0, // Will be calculated
bucket_metrics: HashMap::new(), // Will be populated by scanner
disk_metrics: HashMap::new(), // Will be populated by scanner
}
}
/// Reset all metrics
pub fn reset(&self) {
self.objects_scanned.store(0, Ordering::Relaxed);
self.versions_scanned.store(0, Ordering::Relaxed);
self.directories_scanned.store(0, Ordering::Relaxed);
self.bucket_scans_started.store(0, Ordering::Relaxed);
self.bucket_scans_finished.store(0, Ordering::Relaxed);
self.objects_with_issues.store(0, Ordering::Relaxed);
self.heal_tasks_queued.store(0, Ordering::Relaxed);
self.heal_tasks_completed.store(0, Ordering::Relaxed);
self.heal_tasks_failed.store(0, Ordering::Relaxed);
self.current_cycle.store(0, Ordering::Relaxed);
self.total_cycles.store(0, Ordering::Relaxed);
self.healthy_objects.store(0, Ordering::Relaxed);
self.corrupted_objects.store(0, Ordering::Relaxed);
info!("Scanner metrics reset");
}
}
impl Default for MetricsCollector {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_metrics_collector_creation() {
let collector = MetricsCollector::new();
let metrics = collector.get_metrics();
assert_eq!(metrics.objects_scanned, 0);
assert_eq!(metrics.versions_scanned, 0);
}
#[test]
fn test_metrics_increment() {
let collector = MetricsCollector::new();
collector.increment_objects_scanned(10);
collector.increment_versions_scanned(5);
collector.increment_objects_with_issues(2);
let metrics = collector.get_metrics();
assert_eq!(metrics.objects_scanned, 10);
assert_eq!(metrics.versions_scanned, 5);
assert_eq!(metrics.objects_with_issues, 2);
}
#[test]
fn test_metrics_reset() {
let collector = MetricsCollector::new();
collector.increment_objects_scanned(10);
collector.reset();
let metrics = collector.get_metrics();
assert_eq!(metrics.objects_scanned, 0);
}
}
+555
View File
@@ -0,0 +1,555 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::Result;
use crate::scanner::LoadLevel;
use serde::{Deserialize, Serialize};
use std::{
collections::VecDeque,
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
time::{Duration, SystemTime},
};
use tokio::sync::RwLock;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
/// IO monitor config
#[derive(Debug, Clone)]
pub struct IOMonitorConfig {
/// monitor interval
pub monitor_interval: Duration,
/// history data retention time
pub history_retention: Duration,
/// load evaluation window size
pub load_window_size: usize,
/// whether to enable actual system monitoring
pub enable_system_monitoring: bool,
/// disk path list (for monitoring specific disks)
pub disk_paths: Vec<String>,
}
impl Default for IOMonitorConfig {
fn default() -> Self {
Self {
monitor_interval: Duration::from_secs(1), // 1 second monitor interval
history_retention: Duration::from_secs(300), // keep 5 minutes history
load_window_size: 30, // 30 sample points sliding window
enable_system_monitoring: false, // default use simulated data
disk_paths: Vec::new(),
}
}
}
/// IO monitor metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IOMetrics {
/// timestamp
pub timestamp: SystemTime,
/// disk IOPS (read + write)
pub iops: u64,
/// read IOPS
pub read_iops: u64,
/// write IOPS
pub write_iops: u64,
/// disk queue depth
pub queue_depth: u64,
/// average latency (milliseconds)
pub avg_latency: u64,
/// read latency (milliseconds)
pub read_latency: u64,
/// write latency (milliseconds)
pub write_latency: u64,
/// CPU usage (0-100)
pub cpu_usage: u8,
/// memory usage (0-100)
pub memory_usage: u8,
/// disk usage (0-100)
pub disk_utilization: u8,
/// network IO (Mbps)
pub network_io: u64,
}
impl Default for IOMetrics {
fn default() -> Self {
Self {
timestamp: SystemTime::now(),
iops: 0,
read_iops: 0,
write_iops: 0,
queue_depth: 0,
avg_latency: 0,
read_latency: 0,
write_latency: 0,
cpu_usage: 0,
memory_usage: 0,
disk_utilization: 0,
network_io: 0,
}
}
}
/// load level stats
#[derive(Debug, Clone, Default)]
pub struct LoadLevelStats {
/// low load duration (seconds)
pub low_load_duration: u64,
/// medium load duration (seconds)
pub medium_load_duration: u64,
/// high load duration (seconds)
pub high_load_duration: u64,
/// critical load duration (seconds)
pub critical_load_duration: u64,
/// load transitions
pub load_transitions: u64,
}
/// advanced IO monitor
pub struct AdvancedIOMonitor {
/// config
config: Arc<RwLock<IOMonitorConfig>>,
/// current metrics
current_metrics: Arc<RwLock<IOMetrics>>,
/// history metrics (sliding window)
history_metrics: Arc<RwLock<VecDeque<IOMetrics>>>,
/// current load level
current_load_level: Arc<RwLock<LoadLevel>>,
/// load level history
load_level_history: Arc<RwLock<VecDeque<(SystemTime, LoadLevel)>>>,
/// load level stats
load_stats: Arc<RwLock<LoadLevelStats>>,
/// business IO metrics (updated by external)
business_metrics: Arc<BusinessIOMetrics>,
/// cancel token
cancel_token: CancellationToken,
}
/// business IO metrics
pub struct BusinessIOMetrics {
/// business request latency (milliseconds)
pub request_latency: AtomicU64,
/// business request QPS
pub request_qps: AtomicU64,
/// business error rate (0-10000, 0.00%-100.00%)
pub error_rate: AtomicU64,
/// active connections
pub active_connections: AtomicU64,
/// last update time
pub last_update: Arc<RwLock<SystemTime>>,
}
impl Default for BusinessIOMetrics {
fn default() -> Self {
Self {
request_latency: AtomicU64::new(0),
request_qps: AtomicU64::new(0),
error_rate: AtomicU64::new(0),
active_connections: AtomicU64::new(0),
last_update: Arc::new(RwLock::new(SystemTime::UNIX_EPOCH)),
}
}
}
impl AdvancedIOMonitor {
/// create new advanced IO monitor
pub fn new(config: IOMonitorConfig) -> Self {
Self {
config: Arc::new(RwLock::new(config)),
current_metrics: Arc::new(RwLock::new(IOMetrics::default())),
history_metrics: Arc::new(RwLock::new(VecDeque::new())),
current_load_level: Arc::new(RwLock::new(LoadLevel::Low)),
load_level_history: Arc::new(RwLock::new(VecDeque::new())),
load_stats: Arc::new(RwLock::new(LoadLevelStats::default())),
business_metrics: Arc::new(BusinessIOMetrics::default()),
cancel_token: CancellationToken::new(),
}
}
/// start monitoring
pub async fn start(&self) -> Result<()> {
info!("start advanced IO monitor");
let monitor = self.clone_for_background();
tokio::spawn(async move {
if let Err(e) = monitor.monitoring_loop().await {
error!("IO monitoring loop failed: {}", e);
}
});
Ok(())
}
/// stop monitoring
pub async fn stop(&self) {
info!("stop IO monitor");
self.cancel_token.cancel();
}
/// monitoring loop
async fn monitoring_loop(&self) -> Result<()> {
let mut interval = {
let config = self.config.read().await;
tokio::time::interval(config.monitor_interval)
};
let mut last_load_level = LoadLevel::Low;
let mut load_level_start_time = SystemTime::now();
loop {
tokio::select! {
_ = self.cancel_token.cancelled() => {
info!("IO monitoring loop cancelled");
break;
}
_ = interval.tick() => {
// collect system metrics
let metrics = self.collect_system_metrics().await;
// update current metrics
*self.current_metrics.write().await = metrics.clone();
// update history metrics
self.update_metrics_history(metrics.clone()).await;
// calculate load level
let new_load_level = self.calculate_load_level(&metrics).await;
// check if load level changed
if new_load_level != last_load_level {
self.handle_load_level_change(last_load_level, new_load_level, load_level_start_time).await;
last_load_level = new_load_level;
load_level_start_time = SystemTime::now();
}
// update current load level
*self.current_load_level.write().await = new_load_level;
debug!("IO monitor updated: IOPS={}, queue depth={}, latency={}ms, load level={:?}",
metrics.iops, metrics.queue_depth, metrics.avg_latency, new_load_level);
}
}
}
Ok(())
}
/// collect system metrics
async fn collect_system_metrics(&self) -> IOMetrics {
let config = self.config.read().await;
if config.enable_system_monitoring {
// actual system monitoring implementation
self.collect_real_system_metrics().await
} else {
// simulated data
self.generate_simulated_metrics().await
}
}
/// collect real system metrics (need to be implemented according to specific system)
async fn collect_real_system_metrics(&self) -> IOMetrics {
// TODO: implement actual system metrics collection
// can use procfs, sysfs or other system API
let metrics = IOMetrics {
timestamp: SystemTime::now(),
..Default::default()
};
// example: read /proc/diskstats
if let Ok(diskstats) = tokio::fs::read_to_string("/proc/diskstats").await {
// parse disk stats info
// here need to implement specific parsing logic
debug!("read disk stats info: {} bytes", diskstats.len());
}
// example: read /proc/stat to get CPU info
if let Ok(stat) = tokio::fs::read_to_string("/proc/stat").await {
// parse CPU stats info
debug!("read CPU stats info: {} bytes", stat.len());
}
// example: read /proc/meminfo to get memory info
if let Ok(meminfo) = tokio::fs::read_to_string("/proc/meminfo").await {
// parse memory stats info
debug!("read memory stats info: {} bytes", meminfo.len());
}
metrics
}
/// generate simulated metrics (for testing and development)
async fn generate_simulated_metrics(&self) -> IOMetrics {
use rand::Rng;
let mut rng = rand::rng();
// get business metrics impact
let business_latency = self.business_metrics.request_latency.load(Ordering::Relaxed);
let business_qps = self.business_metrics.request_qps.load(Ordering::Relaxed);
// generate simulated system metrics based on business load
let base_iops = 100 + (business_qps / 10);
let base_latency = 5 + (business_latency / 10);
IOMetrics {
timestamp: SystemTime::now(),
iops: base_iops + rng.random_range(0..50),
read_iops: (base_iops * 6 / 10) + rng.random_range(0..20),
write_iops: (base_iops * 4 / 10) + rng.random_range(0..20),
queue_depth: rng.random_range(1..20),
avg_latency: base_latency + rng.random_range(0..10),
read_latency: base_latency + rng.random_range(0..5),
write_latency: base_latency + rng.random_range(0..15),
cpu_usage: rng.random_range(10..70),
memory_usage: rng.random_range(30..80),
disk_utilization: rng.random_range(20..90),
network_io: rng.random_range(10..1000),
}
}
/// update metrics history
async fn update_metrics_history(&self, metrics: IOMetrics) {
let mut history = self.history_metrics.write().await;
let config = self.config.read().await;
// add new metrics
history.push_back(metrics);
// clean expired data
let retention_cutoff = SystemTime::now() - config.history_retention;
while let Some(front) = history.front() {
if front.timestamp < retention_cutoff {
history.pop_front();
} else {
break;
}
}
// limit window size
while history.len() > config.load_window_size {
history.pop_front();
}
}
/// calculate load level
async fn calculate_load_level(&self, metrics: &IOMetrics) -> LoadLevel {
// multi-dimensional load evaluation algorithm
let mut load_score = 0u32;
// IOPS load evaluation (weight: 25%)
let iops_score = match metrics.iops {
0..=200 => 0,
201..=500 => 15,
501..=1000 => 25,
_ => 35,
};
load_score += iops_score;
// latency load evaluation (weight: 30%)
let latency_score = match metrics.avg_latency {
0..=10 => 0,
11..=50 => 20,
51..=100 => 30,
_ => 40,
};
load_score += latency_score;
// queue depth evaluation (weight: 20%)
let queue_score = match metrics.queue_depth {
0..=5 => 0,
6..=15 => 10,
16..=30 => 20,
_ => 25,
};
load_score += queue_score;
// CPU usage evaluation (weight: 15%)
let cpu_score = match metrics.cpu_usage {
0..=30 => 0,
31..=60 => 8,
61..=80 => 12,
_ => 15,
};
load_score += cpu_score;
// disk usage evaluation (weight: 10%)
let disk_score = match metrics.disk_utilization {
0..=50 => 0,
51..=75 => 5,
76..=90 => 8,
_ => 10,
};
load_score += disk_score;
// business metrics impact
let business_latency = self.business_metrics.request_latency.load(Ordering::Relaxed);
let business_error_rate = self.business_metrics.error_rate.load(Ordering::Relaxed);
if business_latency > 100 {
load_score += 20; // business latency too high
}
if business_error_rate > 100 {
// > 1%
load_score += 15; // business error rate too high
}
// history trend analysis
let trend_score = self.calculate_trend_score().await;
load_score += trend_score;
// determine load level based on total score
match load_score {
0..=30 => LoadLevel::Low,
31..=60 => LoadLevel::Medium,
61..=90 => LoadLevel::High,
_ => LoadLevel::Critical,
}
}
/// calculate trend score
async fn calculate_trend_score(&self) -> u32 {
let history = self.history_metrics.read().await;
if history.len() < 5 {
return 0; // data insufficient, cannot analyze trend
}
// analyze trend of last 5 samples
let recent: Vec<_> = history.iter().rev().take(5).collect();
// check IOPS rising trend
let mut iops_trend = 0;
for i in 1..recent.len() {
if recent[i - 1].iops > recent[i].iops {
iops_trend += 1;
}
}
// check latency rising trend
let mut latency_trend = 0;
for i in 1..recent.len() {
if recent[i - 1].avg_latency > recent[i].avg_latency {
latency_trend += 1;
}
}
// if IOPS and latency are both rising, increase load score
if iops_trend >= 3 && latency_trend >= 3 {
15 // obvious rising trend
} else if iops_trend >= 2 || latency_trend >= 2 {
5 // slight rising trend
} else {
0 // no obvious trend
}
}
/// handle load level change
async fn handle_load_level_change(&self, old_level: LoadLevel, new_level: LoadLevel, start_time: SystemTime) {
let duration = SystemTime::now().duration_since(start_time).unwrap_or(Duration::ZERO);
// update stats
{
let mut stats = self.load_stats.write().await;
match old_level {
LoadLevel::Low => stats.low_load_duration += duration.as_secs(),
LoadLevel::Medium => stats.medium_load_duration += duration.as_secs(),
LoadLevel::High => stats.high_load_duration += duration.as_secs(),
LoadLevel::Critical => stats.critical_load_duration += duration.as_secs(),
}
stats.load_transitions += 1;
}
// update history
{
let mut history = self.load_level_history.write().await;
history.push_back((SystemTime::now(), new_level));
// keep history record in reasonable range
while history.len() > 100 {
history.pop_front();
}
}
info!("load level changed: {:?} -> {:?}, duration: {:?}", old_level, new_level, duration);
// if enter critical load state, record warning
if new_level == LoadLevel::Critical {
warn!("system entered critical load state, Scanner will pause running");
}
}
/// get current load level
pub async fn get_business_load_level(&self) -> LoadLevel {
*self.current_load_level.read().await
}
/// get current metrics
pub async fn get_current_metrics(&self) -> IOMetrics {
self.current_metrics.read().await.clone()
}
/// get history metrics
pub async fn get_history_metrics(&self) -> Vec<IOMetrics> {
self.history_metrics.read().await.iter().cloned().collect()
}
/// get load stats
pub async fn get_load_stats(&self) -> LoadLevelStats {
self.load_stats.read().await.clone()
}
/// update business IO metrics
pub async fn update_business_metrics(&self, latency: u64, qps: u64, error_rate: u64, connections: u64) {
self.business_metrics.request_latency.store(latency, Ordering::Relaxed);
self.business_metrics.request_qps.store(qps, Ordering::Relaxed);
self.business_metrics.error_rate.store(error_rate, Ordering::Relaxed);
self.business_metrics.active_connections.store(connections, Ordering::Relaxed);
*self.business_metrics.last_update.write().await = SystemTime::now();
debug!(
"update business metrics: latency={}ms, QPS={}, error rate={}‰, connections={}",
latency, qps, error_rate, connections
);
}
/// clone for background task
fn clone_for_background(&self) -> Self {
Self {
config: self.config.clone(),
current_metrics: self.current_metrics.clone(),
history_metrics: self.history_metrics.clone(),
current_load_level: self.current_load_level.clone(),
load_level_history: self.load_level_history.clone(),
load_stats: self.load_stats.clone(),
business_metrics: self.business_metrics.clone(),
cancel_token: self.cancel_token.clone(),
}
}
/// reset stats
pub async fn reset_stats(&self) {
*self.load_stats.write().await = LoadLevelStats::default();
self.load_level_history.write().await.clear();
self.history_metrics.write().await.clear();
info!("IO monitor stats reset");
}
/// get load level history
pub async fn get_load_level_history(&self) -> Vec<(SystemTime, LoadLevel)> {
self.load_level_history.read().await.iter().cloned().collect()
}
}
+499
View File
@@ -0,0 +1,499 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::scanner::LoadLevel;
use std::{
sync::{
Arc,
atomic::{AtomicU8, AtomicU64, Ordering},
},
time::{Duration, SystemTime},
};
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
/// IO throttler config
#[derive(Debug, Clone)]
pub struct IOThrottlerConfig {
/// max IOPS limit
pub max_iops: u64,
/// business priority baseline (percentage)
pub base_business_priority: u8,
/// scanner minimum delay (milliseconds)
pub min_scan_delay: u64,
/// scanner maximum delay (milliseconds)
pub max_scan_delay: u64,
/// whether enable dynamic adjustment
pub enable_dynamic_adjustment: bool,
/// adjustment response time (seconds)
pub adjustment_response_time: u64,
}
impl Default for IOThrottlerConfig {
fn default() -> Self {
Self {
max_iops: 1000, // default max 1000 IOPS
base_business_priority: 95, // business priority 95%
min_scan_delay: 5000, // minimum 5s delay
max_scan_delay: 60000, // maximum 60s delay
enable_dynamic_adjustment: true,
adjustment_response_time: 5, // 5 seconds response time
}
}
}
/// resource allocation strategy
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResourceAllocationStrategy {
/// business priority strategy
BusinessFirst,
/// balanced strategy
Balanced,
/// maintenance priority strategy (only used in special cases)
MaintenanceFirst,
}
/// throttle decision
#[derive(Debug, Clone)]
pub struct ThrottleDecision {
/// whether should pause scanning
pub should_pause: bool,
/// suggested scanning delay
pub suggested_delay: Duration,
/// resource allocation suggestion
pub resource_allocation: ResourceAllocation,
/// decision reason
pub reason: String,
}
/// resource allocation
#[derive(Debug, Clone)]
pub struct ResourceAllocation {
/// business IO allocation percentage (0-100)
pub business_percentage: u8,
/// scanner IO allocation percentage (0-100)
pub scanner_percentage: u8,
/// allocation strategy
pub strategy: ResourceAllocationStrategy,
}
/// enhanced IO throttler
///
/// dynamically adjust the resource usage of the scanner based on real-time system load and business demand,
/// ensure business IO gets priority protection.
pub struct AdvancedIOThrottler {
/// config
config: Arc<RwLock<IOThrottlerConfig>>,
/// current IOPS usage (reserved field)
#[allow(dead_code)]
current_iops: Arc<AtomicU64>,
/// business priority weight (0-100)
business_priority: Arc<AtomicU8>,
/// scanning operation delay (milliseconds)
scan_delay: Arc<AtomicU64>,
/// resource allocation strategy
allocation_strategy: Arc<RwLock<ResourceAllocationStrategy>>,
/// throttle history record
throttle_history: Arc<RwLock<Vec<ThrottleRecord>>>,
/// last adjustment time (reserved field)
#[allow(dead_code)]
last_adjustment: Arc<RwLock<SystemTime>>,
}
/// throttle record
#[derive(Debug, Clone)]
pub struct ThrottleRecord {
/// timestamp
pub timestamp: SystemTime,
/// load level
pub load_level: LoadLevel,
/// decision
pub decision: ThrottleDecision,
/// system metrics snapshot
pub metrics_snapshot: MetricsSnapshot,
}
/// metrics snapshot
#[derive(Debug, Clone)]
pub struct MetricsSnapshot {
/// IOPS
pub iops: u64,
/// latency
pub latency: u64,
/// CPU usage
pub cpu_usage: u8,
/// memory usage
pub memory_usage: u8,
}
impl AdvancedIOThrottler {
/// create new advanced IO throttler
pub fn new(config: IOThrottlerConfig) -> Self {
Self {
config: Arc::new(RwLock::new(config)),
current_iops: Arc::new(AtomicU64::new(0)),
business_priority: Arc::new(AtomicU8::new(95)),
scan_delay: Arc::new(AtomicU64::new(5000)),
allocation_strategy: Arc::new(RwLock::new(ResourceAllocationStrategy::BusinessFirst)),
throttle_history: Arc::new(RwLock::new(Vec::new())),
last_adjustment: Arc::new(RwLock::new(SystemTime::UNIX_EPOCH)),
}
}
/// adjust scanning delay based on load level
pub async fn adjust_for_load_level(&self, load_level: LoadLevel) -> Duration {
let config = self.config.read().await;
let delay_ms = match load_level {
LoadLevel::Low => {
// low load: use minimum delay
self.scan_delay.store(config.min_scan_delay, Ordering::Relaxed);
self.business_priority
.store(config.base_business_priority.saturating_sub(5), Ordering::Relaxed);
config.min_scan_delay
}
LoadLevel::Medium => {
// medium load: increase delay moderately
let delay = config.min_scan_delay * 5; // 500ms
self.scan_delay.store(delay, Ordering::Relaxed);
self.business_priority.store(config.base_business_priority, Ordering::Relaxed);
delay
}
LoadLevel::High => {
// high load: increase delay significantly
let delay = config.min_scan_delay * 10; // 50s
self.scan_delay.store(delay, Ordering::Relaxed);
self.business_priority
.store(config.base_business_priority.saturating_add(3), Ordering::Relaxed);
delay
}
LoadLevel::Critical => {
// critical load: maximum delay or pause
let delay = config.max_scan_delay; // 60s
self.scan_delay.store(delay, Ordering::Relaxed);
self.business_priority.store(99, Ordering::Relaxed);
delay
}
};
let duration = Duration::from_millis(delay_ms);
debug!("Adjust scanning delay based on load level {:?}: {:?}", load_level, duration);
duration
}
/// create throttle decision
pub async fn make_throttle_decision(&self, load_level: LoadLevel, metrics: Option<MetricsSnapshot>) -> ThrottleDecision {
let _config = self.config.read().await;
let should_pause = matches!(load_level, LoadLevel::Critical);
let suggested_delay = self.adjust_for_load_level(load_level).await;
let resource_allocation = self.calculate_resource_allocation(load_level).await;
let reason = match load_level {
LoadLevel::Low => "system load is low, scanner can run normally".to_string(),
LoadLevel::Medium => "system load is moderate, scanner is running at reduced speed".to_string(),
LoadLevel::High => "system load is high, scanner is running at significantly reduced speed".to_string(),
LoadLevel::Critical => "system load is too high, scanner is paused".to_string(),
};
let decision = ThrottleDecision {
should_pause,
suggested_delay,
resource_allocation,
reason,
};
// record decision history
if let Some(snapshot) = metrics {
self.record_throttle_decision(load_level, decision.clone(), snapshot).await;
}
decision
}
/// calculate resource allocation
async fn calculate_resource_allocation(&self, load_level: LoadLevel) -> ResourceAllocation {
let strategy = *self.allocation_strategy.read().await;
let (business_pct, scanner_pct) = match (strategy, load_level) {
(ResourceAllocationStrategy::BusinessFirst, LoadLevel::Low) => (90, 10),
(ResourceAllocationStrategy::BusinessFirst, LoadLevel::Medium) => (95, 5),
(ResourceAllocationStrategy::BusinessFirst, LoadLevel::High) => (98, 2),
(ResourceAllocationStrategy::BusinessFirst, LoadLevel::Critical) => (99, 1),
(ResourceAllocationStrategy::Balanced, LoadLevel::Low) => (80, 20),
(ResourceAllocationStrategy::Balanced, LoadLevel::Medium) => (85, 15),
(ResourceAllocationStrategy::Balanced, LoadLevel::High) => (90, 10),
(ResourceAllocationStrategy::Balanced, LoadLevel::Critical) => (95, 5),
(ResourceAllocationStrategy::MaintenanceFirst, _) => (70, 30), // special maintenance mode
};
ResourceAllocation {
business_percentage: business_pct,
scanner_percentage: scanner_pct,
strategy,
}
}
/// check whether should pause scanning
pub async fn should_pause_scanning(&self, load_level: LoadLevel) -> bool {
match load_level {
LoadLevel::Critical => {
warn!("System load reached critical level, pausing scanner");
true
}
_ => false,
}
}
/// record throttle decision
async fn record_throttle_decision(&self, load_level: LoadLevel, decision: ThrottleDecision, metrics: MetricsSnapshot) {
let record = ThrottleRecord {
timestamp: SystemTime::now(),
load_level,
decision,
metrics_snapshot: metrics,
};
let mut history = self.throttle_history.write().await;
history.push(record);
// keep history record in reasonable range (last 1000 records)
while history.len() > 1000 {
history.remove(0);
}
}
/// set resource allocation strategy
pub async fn set_allocation_strategy(&self, strategy: ResourceAllocationStrategy) {
*self.allocation_strategy.write().await = strategy;
info!("Set resource allocation strategy: {:?}", strategy);
}
/// get current resource allocation
pub async fn get_current_allocation(&self) -> ResourceAllocation {
let current_load = LoadLevel::Low; // need to get from external
self.calculate_resource_allocation(current_load).await
}
/// get throttle history
pub async fn get_throttle_history(&self) -> Vec<ThrottleRecord> {
self.throttle_history.read().await.clone()
}
/// get throttle stats
pub async fn get_throttle_stats(&self) -> ThrottleStats {
let history = self.throttle_history.read().await;
let total_decisions = history.len();
let pause_decisions = history.iter().filter(|r| r.decision.should_pause).count();
let mut delay_sum = Duration::ZERO;
for record in history.iter() {
delay_sum += record.decision.suggested_delay;
}
let avg_delay = if total_decisions > 0 {
delay_sum / total_decisions as u32
} else {
Duration::ZERO
};
// count by load level
let low_count = history.iter().filter(|r| r.load_level == LoadLevel::Low).count();
let medium_count = history.iter().filter(|r| r.load_level == LoadLevel::Medium).count();
let high_count = history.iter().filter(|r| r.load_level == LoadLevel::High).count();
let critical_count = history.iter().filter(|r| r.load_level == LoadLevel::Critical).count();
ThrottleStats {
total_decisions,
pause_decisions,
average_delay: avg_delay,
load_level_distribution: LoadLevelDistribution {
low_count,
medium_count,
high_count,
critical_count,
},
}
}
/// reset throttle history
pub async fn reset_history(&self) {
self.throttle_history.write().await.clear();
info!("Reset throttle history");
}
/// update config
pub async fn update_config(&self, new_config: IOThrottlerConfig) {
*self.config.write().await = new_config;
info!("Updated IO throttler configuration");
}
/// get current scanning delay
pub fn get_current_scan_delay(&self) -> Duration {
let delay_ms = self.scan_delay.load(Ordering::Relaxed);
Duration::from_millis(delay_ms)
}
/// get current business priority
pub fn get_current_business_priority(&self) -> u8 {
self.business_priority.load(Ordering::Relaxed)
}
/// simulate business load pressure test
pub async fn simulate_business_pressure(&self, duration: Duration) -> SimulationResult {
info!("Start simulating business load pressure test, duration: {:?}", duration);
let start_time = SystemTime::now();
let mut simulation_records = Vec::new();
// simulate different load level changes
let load_levels = [
LoadLevel::Low,
LoadLevel::Medium,
LoadLevel::High,
LoadLevel::Critical,
LoadLevel::High,
LoadLevel::Medium,
LoadLevel::Low,
];
let step_duration = duration / load_levels.len() as u32;
for (i, &load_level) in load_levels.iter().enumerate() {
let _step_start = SystemTime::now();
// simulate metrics for this load level
let metrics = MetricsSnapshot {
iops: match load_level {
LoadLevel::Low => 200,
LoadLevel::Medium => 500,
LoadLevel::High => 800,
LoadLevel::Critical => 1200,
},
latency: match load_level {
LoadLevel::Low => 10,
LoadLevel::Medium => 25,
LoadLevel::High => 60,
LoadLevel::Critical => 150,
},
cpu_usage: match load_level {
LoadLevel::Low => 30,
LoadLevel::Medium => 50,
LoadLevel::High => 75,
LoadLevel::Critical => 95,
},
memory_usage: match load_level {
LoadLevel::Low => 40,
LoadLevel::Medium => 60,
LoadLevel::High => 80,
LoadLevel::Critical => 90,
},
};
let decision = self.make_throttle_decision(load_level, Some(metrics.clone())).await;
simulation_records.push(SimulationRecord {
step: i + 1,
load_level,
metrics,
decision: decision.clone(),
step_duration,
});
info!(
"simulate step {}: load={:?}, delay={:?}, pause={}",
i + 1,
load_level,
decision.suggested_delay,
decision.should_pause
);
// wait for step duration
tokio::time::sleep(step_duration).await;
}
let total_duration = SystemTime::now().duration_since(start_time).unwrap_or(Duration::ZERO);
SimulationResult {
total_duration,
simulation_records,
final_stats: self.get_throttle_stats().await,
}
}
}
/// throttle stats
#[derive(Debug, Clone)]
pub struct ThrottleStats {
/// total decisions
pub total_decisions: usize,
/// pause decisions
pub pause_decisions: usize,
/// average delay
pub average_delay: Duration,
/// load level distribution
pub load_level_distribution: LoadLevelDistribution,
}
/// load level distribution
#[derive(Debug, Clone)]
pub struct LoadLevelDistribution {
/// low load count
pub low_count: usize,
/// medium load count
pub medium_count: usize,
/// high load count
pub high_count: usize,
/// critical load count
pub critical_count: usize,
}
/// simulation result
#[derive(Debug, Clone)]
pub struct SimulationResult {
/// total duration
pub total_duration: Duration,
/// simulation records
pub simulation_records: Vec<SimulationRecord>,
/// final stats
pub final_stats: ThrottleStats,
}
/// simulation record
#[derive(Debug, Clone)]
pub struct SimulationRecord {
/// step number
pub step: usize,
/// load level
pub load_level: LoadLevel,
/// metrics snapshot
pub metrics: MetricsSnapshot,
/// throttle decision
pub decision: ThrottleDecision,
/// step duration
pub step_duration: Duration,
}
impl Default for AdvancedIOThrottler {
fn default() -> Self {
Self::new(IOThrottlerConfig::default())
}
}
+269
View File
@@ -0,0 +1,269 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::Result;
use rustfs_common::data_usage::SizeSummary;
use rustfs_common::metrics::IlmAction;
use rustfs_ecstore::bucket::{
lifecycle::{
bucket_lifecycle_audit::LcEventSrc,
bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_lifecycle_action, eval_action_from_lifecycle},
lifecycle,
lifecycle::Lifecycle,
},
metadata_sys::get_object_lock_config,
object_lock::objectlock_sys::{BucketObjectLockSys, enforce_retention_for_deletion},
versioning::VersioningApi,
versioning_sys::BucketVersioningSys,
};
use rustfs_ecstore::store_api::{ObjectInfo, ObjectToDelete};
use rustfs_filemeta::FileInfo;
use s3s::dto::{BucketLifecycleConfiguration as LifecycleConfig, VersioningConfiguration};
use std::sync::{
Arc,
atomic::{AtomicU64, Ordering},
};
use time::OffsetDateTime;
use tracing::info;
static SCANNER_EXCESS_OBJECT_VERSIONS: AtomicU64 = AtomicU64::new(100);
static SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE: AtomicU64 = AtomicU64::new(1024 * 1024 * 1024 * 1024); // 1 TB
#[derive(Clone)]
pub struct ScannerItem {
pub bucket: String,
pub object_name: String,
pub lifecycle: Option<Arc<LifecycleConfig>>,
pub versioning: Option<Arc<VersioningConfiguration>>,
}
impl ScannerItem {
pub fn new(
bucket: String,
lifecycle: Option<Arc<LifecycleConfig>>,
versioning: Option<Arc<VersioningConfiguration>>,
) -> Self {
Self {
bucket,
object_name: "".to_string(),
lifecycle,
versioning,
}
}
pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?;
if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst) as usize {
// todo
}
let mut cumulative_size = 0;
for obj_info in obj_infos.iter() {
cumulative_size += obj_info.size;
}
if cumulative_size >= SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE.load(Ordering::SeqCst) as i64 {
//todo
}
Ok(obj_infos)
}
pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await {
rcfg.mode.is_some()
} else {
false
};
let _vcfg = BucketVersioningSys::get(&self.bucket).await?;
let versioned = match BucketVersioningSys::get(&self.bucket).await {
Ok(vcfg) => vcfg.versioned(&self.object_name),
Err(_) => false,
};
let mut object_infos = Vec::with_capacity(fivs.len());
if self.lifecycle.is_none() {
for info in fivs.iter() {
object_infos.push(ObjectInfo::from_file_info(info, &self.bucket, &self.object_name, versioned));
}
return Ok(object_infos);
}
let event = self
.lifecycle
.as_ref()
.expect("lifecycle err.")
.clone()
.noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts {
name: self.object_name.clone(),
..Default::default()
})
.await;
let lim = event.newer_noncurrent_versions;
if lim == 0 || fivs.len() <= lim + 1 {
for fi in fivs.iter() {
object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_name, versioned));
}
return Ok(object_infos);
}
let overflow_versions = &fivs[lim + 1..];
for fi in fivs[..lim + 1].iter() {
object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_name, versioned));
}
let mut to_del = Vec::<ObjectToDelete>::with_capacity(overflow_versions.len());
for fi in overflow_versions.iter() {
let obj = ObjectInfo::from_file_info(fi, &self.bucket, &self.object_name, versioned);
if lock_enabled && enforce_retention_for_deletion(&obj) {
//if enforce_retention_for_deletion(&obj) {
/*if self.debug {
if obj.version_id.is_some() {
info!("lifecycle: {} v({}) is locked, not deleting\n", obj.name, obj.version_id.expect("err"));
} else {
info!("lifecycle: {} is locked, not deleting\n", obj.name);
}
}*/
object_infos.push(obj);
continue;
}
if OffsetDateTime::now_utc().unix_timestamp()
< lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32)
.unix_timestamp()
{
object_infos.push(obj);
continue;
}
to_del.push(ObjectToDelete {
object_name: obj.name,
version_id: obj.version_id,
..Default::default()
});
}
if !to_del.is_empty() {
let mut expiry_state = GLOBAL_ExpiryState.write().await;
expiry_state.enqueue_by_newer_noncurrent(&self.bucket, to_del, event).await;
}
Ok(object_infos)
}
pub async fn apply_actions(&mut self, oi: &ObjectInfo, _size_s: &mut SizeSummary) -> (bool, i64) {
let (action, _size) = self.apply_lifecycle(oi).await;
info!(
"apply_actions {} {} {:?} {:?}",
oi.bucket.clone(),
oi.name.clone(),
oi.version_id.clone(),
oi.user_defined.clone()
);
// Create a mutable clone if you need to modify fields
/*let mut oi = oi.clone();
oi.replication_status = ReplicationStatusType::from(
oi.user_defined
.get("x-amz-bucket-replication-status")
.unwrap_or(&"PENDING".to_string()),
);
info!("apply status is: {:?}", oi.replication_status);
self.heal_replication(&oi, _size_s).await;*/
if action.delete_all() {
return (true, 0);
}
(false, oi.size)
}
async fn apply_lifecycle(&mut self, oi: &ObjectInfo) -> (IlmAction, i64) {
let size = oi.size;
if self.lifecycle.is_none() {
info!("apply_lifecycle: No lifecycle config for object: {}", oi.name);
return (IlmAction::NoneAction, size);
}
info!("apply_lifecycle: Lifecycle config exists for object: {}", oi.name);
let (olcfg, rcfg) = if self.bucket != ".minio.sys" {
(
get_object_lock_config(&self.bucket).await.ok(),
None, // FIXME: replication config
)
} else {
(None, None)
};
info!("apply_lifecycle: Evaluating lifecycle for object: {}", oi.name);
let lifecycle = match self.lifecycle.as_ref() {
Some(lc) => lc,
None => {
info!("No lifecycle configuration found for object: {}", oi.name);
return (IlmAction::NoneAction, 0);
}
};
let lc_evt = eval_action_from_lifecycle(
lifecycle,
olcfg
.as_ref()
.and_then(|(c, _)| c.rule.as_ref().and_then(|r| r.default_retention.clone())),
rcfg.clone(),
oi, // Pass oi directly
)
.await;
info!("lifecycle: {} Initial scan: {} (action: {:?})", oi.name, lc_evt.action, lc_evt.action);
let mut new_size = size;
match lc_evt.action {
IlmAction::DeleteVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
info!("apply_lifecycle: Object {} marked for version deletion, new_size=0", oi.name);
new_size = 0;
}
IlmAction::DeleteAction => {
info!("apply_lifecycle: Object {} marked for deletion", oi.name);
if let Some(vcfg) = &self.versioning {
if !vcfg.enabled() {
info!("apply_lifecycle: Versioning disabled, setting new_size=0");
new_size = 0;
}
} else {
info!("apply_lifecycle: No versioning config, setting new_size=0");
new_size = 0;
}
}
IlmAction::NoneAction => {
info!("apply_lifecycle: No action for object {}", oi.name);
}
_ => {
info!("apply_lifecycle: Other action {:?} for object {}", lc_evt.action, oi.name);
}
}
if lc_evt.action != IlmAction::NoneAction {
info!("apply_lifecycle: Applying lifecycle action {:?} for object {}", lc_evt.action, oi.name);
apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await;
} else {
info!("apply_lifecycle: Skipping lifecycle action for object {} as no action is needed", oi.name);
}
(lc_evt.action, new_size)
}
}
+675
View File
@@ -0,0 +1,675 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{Error, Result};
use rustfs_common::data_usage::DiskUsageStatus;
use rustfs_ecstore::data_usage::{
LocalUsageSnapshot, LocalUsageSnapshotMeta, data_usage_state_dir, ensure_data_usage_layout, snapshot_file_name,
write_local_snapshot,
};
use rustfs_ecstore::disk::DiskAPI;
use rustfs_ecstore::store::ECStore;
use rustfs_ecstore::store_api::ObjectInfo;
use rustfs_filemeta::{FileInfo, FileMeta, FileMetaVersion, VersionType};
use serde::{Deserialize, Serialize};
use serde_json::{from_slice, to_vec};
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::{fs, task};
use tracing::warn;
use walkdir::WalkDir;
const STATE_FILE_EXTENSION: &str = "";
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct LocalObjectUsage {
pub bucket: String,
pub object: String,
pub last_modified_ns: Option<i128>,
pub versions_count: u64,
pub delete_markers_count: u64,
pub total_size: u64,
pub has_live_object: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
struct IncrementalScanState {
last_scan_ns: Option<i128>,
objects: HashMap<String, LocalObjectUsage>,
}
struct DiskScanResult {
snapshot: LocalUsageSnapshot,
state: IncrementalScanState,
objects_by_bucket: HashMap<String, Vec<LocalObjectRecord>>,
status: DiskUsageStatus,
}
#[derive(Debug, Clone)]
pub struct LocalObjectRecord {
pub usage: LocalObjectUsage,
pub object_info: Option<rustfs_ecstore::store_api::ObjectInfo>,
}
#[derive(Debug, Default)]
pub struct LocalScanOutcome {
pub snapshots: Vec<LocalUsageSnapshot>,
pub bucket_objects: HashMap<String, Vec<LocalObjectRecord>>,
pub disk_status: Vec<DiskUsageStatus>,
}
/// Scan all local primary disks and persist refreshed usage snapshots.
pub async fn scan_and_persist_local_usage(store: Arc<ECStore>) -> Result<LocalScanOutcome> {
let mut snapshots = Vec::new();
let mut bucket_objects: HashMap<String, Vec<LocalObjectRecord>> = HashMap::new();
let mut disk_status = Vec::new();
for (pool_idx, pool) in store.pools.iter().enumerate() {
for set_disks in pool.disk_set.iter() {
let disks = {
let guard = set_disks.disks.read().await;
guard.clone()
};
for (disk_index, disk_opt) in disks.into_iter().enumerate() {
let Some(disk) = disk_opt else {
continue;
};
if !disk.is_local() {
continue;
}
// Count objects once by scanning only disk index zero from each set.
if disk_index != 0 {
continue;
}
let disk_id = match disk.get_disk_id().await.map_err(Error::from)? {
Some(id) => id.to_string(),
None => {
warn!("Skipping disk without ID: {}", disk.to_string());
continue;
}
};
let root = disk.path();
ensure_data_usage_layout(root.as_path()).await.map_err(Error::from)?;
let meta = LocalUsageSnapshotMeta {
disk_id: disk_id.clone(),
pool_index: Some(pool_idx),
set_index: Some(set_disks.set_index),
disk_index: Some(disk_index),
};
let state_path = state_file_path(root.as_path(), &disk_id);
let state = read_scan_state(&state_path).await?;
let root_clone = root.clone();
let meta_clone = meta.clone();
let handle = task::spawn_blocking(move || scan_disk_blocking(root_clone, meta_clone, state));
match handle.await {
Ok(Ok(result)) => {
write_local_snapshot(root.as_path(), &disk_id, &result.snapshot)
.await
.map_err(Error::from)?;
write_scan_state(&state_path, &result.state).await?;
snapshots.push(result.snapshot);
for (bucket, records) in result.objects_by_bucket {
bucket_objects.entry(bucket).or_default().extend(records.into_iter());
}
disk_status.push(result.status);
}
Ok(Err(err)) => {
warn!("Failed to scan disk {}: {}", disk.to_string(), err);
}
Err(join_err) => {
warn!("Disk scan task panicked for disk {}: {}", disk.to_string(), join_err);
}
}
}
}
}
Ok(LocalScanOutcome {
snapshots,
bucket_objects,
disk_status,
})
}
fn scan_disk_blocking(root: PathBuf, meta: LocalUsageSnapshotMeta, mut state: IncrementalScanState) -> Result<DiskScanResult> {
let now = SystemTime::now();
let now_ns = system_time_to_ns(now);
let mut visited: HashSet<String> = HashSet::new();
let mut emitted: HashSet<String> = HashSet::new();
let mut objects_by_bucket: HashMap<String, Vec<LocalObjectRecord>> = HashMap::new();
let mut status = DiskUsageStatus {
disk_id: meta.disk_id.clone(),
pool_index: meta.pool_index,
set_index: meta.set_index,
disk_index: meta.disk_index,
last_update: None,
snapshot_exists: false,
};
for entry in WalkDir::new(&root).follow_links(false).into_iter().filter_map(|res| res.ok()) {
if !entry.file_type().is_file() {
continue;
}
if entry.file_name() != "xl.meta" {
continue;
}
let xl_path = entry.path().to_path_buf();
let Some(object_dir) = xl_path.parent() else {
continue;
};
let Some(rel_path) = object_dir.strip_prefix(&root).ok().map(normalize_path) else {
continue;
};
let mut components = rel_path.split('/');
let Some(bucket_name) = components.next() else {
continue;
};
if bucket_name.starts_with('.') {
continue;
}
let object_key = components.collect::<Vec<_>>().join("/");
visited.insert(rel_path.clone());
let metadata = match std::fs::metadata(&xl_path) {
Ok(meta) => meta,
Err(err) => {
warn!("Failed to read metadata for {xl_path:?}: {err}");
continue;
}
};
let mtime_ns = metadata.modified().ok().map(system_time_to_ns);
let should_parse = match state.objects.get(&rel_path) {
Some(existing) => existing.last_modified_ns != mtime_ns,
None => true,
};
if should_parse {
match std::fs::read(&xl_path) {
Ok(buf) => match FileMeta::load(&buf) {
Ok(file_meta) => match compute_object_usage(bucket_name, object_key.as_str(), &file_meta) {
Ok(Some(mut record)) => {
record.usage.last_modified_ns = mtime_ns;
state.objects.insert(rel_path.clone(), record.usage.clone());
emitted.insert(rel_path.clone());
objects_by_bucket.entry(record.usage.bucket.clone()).or_default().push(record);
}
Ok(None) => {
state.objects.remove(&rel_path);
}
Err(err) => {
warn!("Failed to parse usage from {:?}: {}", xl_path, err);
}
},
Err(err) => {
warn!("Failed to decode xl.meta {:?}: {}", xl_path, err);
}
},
Err(err) => {
warn!("Failed to read xl.meta {:?}: {}", xl_path, err);
}
}
}
}
state.objects.retain(|key, _| visited.contains(key));
state.last_scan_ns = Some(now_ns);
for (key, usage) in &state.objects {
if emitted.contains(key) {
continue;
}
objects_by_bucket
.entry(usage.bucket.clone())
.or_default()
.push(LocalObjectRecord {
usage: usage.clone(),
object_info: None,
});
}
let snapshot = build_snapshot(meta, &state.objects, now);
status.snapshot_exists = true;
status.last_update = Some(now);
Ok(DiskScanResult {
snapshot,
state,
objects_by_bucket,
status,
})
}
fn compute_object_usage(bucket: &str, object: &str, file_meta: &FileMeta) -> Result<Option<LocalObjectRecord>> {
let mut versions_count = 0u64;
let mut delete_markers_count = 0u64;
let mut total_size = 0u64;
let mut has_live_object = false;
let mut latest_file_info: Option<FileInfo> = None;
for shallow in &file_meta.versions {
match shallow.header.version_type {
VersionType::Object => {
let version = match FileMetaVersion::try_from(shallow.meta.as_slice()) {
Ok(version) => version,
Err(err) => {
warn!("Failed to parse file meta version: {}", err);
continue;
}
};
if let Some(obj) = version.object {
if !has_live_object {
total_size = obj.size.max(0) as u64;
}
has_live_object = true;
versions_count = versions_count.saturating_add(1);
if latest_file_info.is_none() {
if let Ok(info) = file_meta.into_fileinfo(bucket, object, "", false, false) {
latest_file_info = Some(info);
}
}
}
}
VersionType::Delete => {
delete_markers_count = delete_markers_count.saturating_add(1);
versions_count = versions_count.saturating_add(1);
}
_ => {}
}
}
if !has_live_object && delete_markers_count == 0 {
return Ok(None);
}
let object_info = latest_file_info.as_ref().map(|fi| {
let versioned = fi.version_id.is_some();
ObjectInfo::from_file_info(fi, bucket, object, versioned)
});
Ok(Some(LocalObjectRecord {
usage: LocalObjectUsage {
bucket: bucket.to_string(),
object: object.to_string(),
last_modified_ns: None,
versions_count,
delete_markers_count,
total_size,
has_live_object,
},
object_info,
}))
}
fn build_snapshot(
meta: LocalUsageSnapshotMeta,
objects: &HashMap<String, LocalObjectUsage>,
now: SystemTime,
) -> LocalUsageSnapshot {
let mut snapshot = LocalUsageSnapshot::new(meta);
for usage in objects.values() {
let bucket_entry = snapshot.buckets_usage.entry(usage.bucket.clone()).or_default();
if usage.has_live_object {
bucket_entry.objects_count = bucket_entry.objects_count.saturating_add(1);
}
bucket_entry.versions_count = bucket_entry.versions_count.saturating_add(usage.versions_count);
bucket_entry.delete_markers_count = bucket_entry.delete_markers_count.saturating_add(usage.delete_markers_count);
bucket_entry.size = bucket_entry.size.saturating_add(usage.total_size);
}
snapshot.last_update = Some(now);
snapshot.recompute_totals();
snapshot
}
fn normalize_path(path: &Path) -> String {
path.iter()
.map(|component| component.to_string_lossy())
.collect::<Vec<_>>()
.join("/")
}
fn system_time_to_ns(time: SystemTime) -> i128 {
match time.duration_since(UNIX_EPOCH) {
Ok(duration) => {
let secs = duration.as_secs() as i128;
let nanos = duration.subsec_nanos() as i128;
secs * 1_000_000_000 + nanos
}
Err(err) => {
let duration = err.duration();
let secs = duration.as_secs() as i128;
let nanos = duration.subsec_nanos() as i128;
-(secs * 1_000_000_000 + nanos)
}
}
}
fn state_file_path(root: &Path, disk_id: &str) -> PathBuf {
let mut path = data_usage_state_dir(root);
path.push(format!("{}{}", snapshot_file_name(disk_id), STATE_FILE_EXTENSION));
path
}
async fn read_scan_state(path: &Path) -> Result<IncrementalScanState> {
match fs::read(path).await {
Ok(bytes) => from_slice(&bytes).map_err(|err| Error::Serialization(err.to_string())),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(IncrementalScanState::default()),
Err(err) => Err(err.into()),
}
}
async fn write_scan_state(path: &Path, state: &IncrementalScanState) -> Result<()> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
let data = to_vec(state).map_err(|err| Error::Serialization(err.to_string()))?;
fs::write(path, data).await?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_filemeta::{ChecksumAlgo, ErasureAlgo, FileMetaShallowVersion, MetaDeleteMarker, MetaObject};
use std::collections::HashMap;
use std::fs;
use tempfile::TempDir;
use time::OffsetDateTime;
use uuid::Uuid;
fn build_file_meta_with_object(erasure_index: usize, size: i64) -> FileMeta {
let mut file_meta = FileMeta::default();
let meta_object = MetaObject {
version_id: Some(Uuid::new_v4()),
data_dir: Some(Uuid::new_v4()),
erasure_algorithm: ErasureAlgo::ReedSolomon,
erasure_m: 2,
erasure_n: 2,
erasure_block_size: 4096,
erasure_index,
erasure_dist: vec![0_u8, 1, 2, 3],
bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
part_numbers: vec![1],
part_etags: vec!["etag".to_string()],
part_sizes: vec![size as usize],
part_actual_sizes: vec![size],
part_indices: Vec::new(),
size,
mod_time: Some(OffsetDateTime::now_utc()),
meta_sys: HashMap::new(),
meta_user: HashMap::new(),
};
let version = FileMetaVersion {
version_type: VersionType::Object,
object: Some(meta_object),
delete_marker: None,
write_version: 1,
};
let shallow = FileMetaShallowVersion::try_from(version).expect("convert version");
file_meta.versions.push(shallow);
file_meta
}
fn build_file_meta_with_delete_marker() -> FileMeta {
let mut file_meta = FileMeta::default();
let delete_marker = MetaDeleteMarker {
version_id: Some(Uuid::new_v4()),
mod_time: Some(OffsetDateTime::now_utc()),
meta_sys: HashMap::new(),
};
let version = FileMetaVersion {
version_type: VersionType::Delete,
object: None,
delete_marker: Some(delete_marker),
write_version: 2,
};
let shallow = FileMetaShallowVersion::try_from(version).expect("convert delete marker");
file_meta.versions.push(shallow);
file_meta
}
#[test]
fn compute_object_usage_primary_disk() {
let file_meta = build_file_meta_with_object(0, 1024);
let record = compute_object_usage("bucket", "foo/bar", &file_meta)
.expect("compute usage")
.expect("record should exist");
assert!(record.usage.has_live_object);
assert_eq!(record.usage.bucket, "bucket");
assert_eq!(record.usage.object, "foo/bar");
assert_eq!(record.usage.total_size, 1024);
assert!(record.object_info.is_some(), "object info should be synthesized");
}
#[test]
fn compute_object_usage_handles_non_primary_disk() {
let file_meta = build_file_meta_with_object(1, 2048);
let record = compute_object_usage("bucket", "obj", &file_meta)
.expect("compute usage")
.expect("record should exist for non-primary shard");
assert!(record.usage.has_live_object);
}
#[test]
fn compute_object_usage_reports_delete_marker() {
let file_meta = build_file_meta_with_delete_marker();
let record = compute_object_usage("bucket", "obj", &file_meta)
.expect("compute usage")
.expect("delete marker record");
assert!(!record.usage.has_live_object);
assert_eq!(record.usage.delete_markers_count, 1);
assert_eq!(record.usage.versions_count, 1);
}
#[test]
fn build_snapshot_accumulates_usage() {
let mut objects = HashMap::new();
objects.insert(
"bucket/a".to_string(),
LocalObjectUsage {
bucket: "bucket".to_string(),
object: "a".to_string(),
last_modified_ns: None,
versions_count: 2,
delete_markers_count: 1,
total_size: 512,
has_live_object: true,
},
);
let snapshot = build_snapshot(LocalUsageSnapshotMeta::default(), &objects, SystemTime::now());
let usage = snapshot.buckets_usage.get("bucket").expect("bucket entry should exist");
assert_eq!(usage.objects_count, 1);
assert_eq!(usage.versions_count, 2);
assert_eq!(usage.delete_markers_count, 1);
assert_eq!(usage.size, 512);
}
#[test]
fn scan_disk_blocking_handles_incremental_updates() {
let temp_dir = TempDir::new().expect("create temp dir");
let root = temp_dir.path();
let bucket_dir = root.join("bench");
let object1_dir = bucket_dir.join("obj1");
fs::create_dir_all(&object1_dir).expect("create first object directory");
let file_meta = build_file_meta_with_object(0, 1024);
let bytes = file_meta.marshal_msg().expect("serialize first object");
fs::write(object1_dir.join("xl.meta"), bytes).expect("write first xl.meta");
let meta = LocalUsageSnapshotMeta {
disk_id: "disk-test".to_string(),
..Default::default()
};
let DiskScanResult {
snapshot: snapshot1,
state,
..
} = scan_disk_blocking(root.to_path_buf(), meta.clone(), IncrementalScanState::default()).expect("initial scan succeeds");
let usage1 = snapshot1.buckets_usage.get("bench").expect("bucket stats recorded");
assert_eq!(usage1.objects_count, 1);
assert_eq!(usage1.size, 1024);
assert_eq!(state.objects.len(), 1);
let object2_dir = bucket_dir.join("nested").join("obj2");
fs::create_dir_all(&object2_dir).expect("create second object directory");
let second_meta = build_file_meta_with_object(0, 2048);
let bytes = second_meta.marshal_msg().expect("serialize second object");
fs::write(object2_dir.join("xl.meta"), bytes).expect("write second xl.meta");
let DiskScanResult {
snapshot: snapshot2,
state: state_next,
..
} = scan_disk_blocking(root.to_path_buf(), meta.clone(), state).expect("incremental scan succeeds");
let usage2 = snapshot2
.buckets_usage
.get("bench")
.expect("bucket stats recorded after addition");
assert_eq!(usage2.objects_count, 2);
assert_eq!(usage2.size, 1024 + 2048);
assert_eq!(state_next.objects.len(), 2);
fs::remove_dir_all(&object1_dir).expect("remove first object");
let DiskScanResult {
snapshot: snapshot3,
state: state_final,
..
} = scan_disk_blocking(root.to_path_buf(), meta, state_next).expect("scan after deletion succeeds");
let usage3 = snapshot3
.buckets_usage
.get("bench")
.expect("bucket stats recorded after deletion");
assert_eq!(usage3.objects_count, 1);
assert_eq!(usage3.size, 2048);
assert_eq!(state_final.objects.len(), 1);
assert!(
state_final.objects.keys().all(|path| path.contains("nested")),
"state should only keep surviving object"
);
}
#[test]
fn scan_disk_blocking_recovers_from_stale_state_entries() {
let temp_dir = TempDir::new().expect("create temp dir");
let root = temp_dir.path();
let mut stale_state = IncrementalScanState::default();
stale_state.objects.insert(
"bench/stale".to_string(),
LocalObjectUsage {
bucket: "bench".to_string(),
object: "stale".to_string(),
last_modified_ns: Some(42),
versions_count: 1,
delete_markers_count: 0,
total_size: 512,
has_live_object: true,
},
);
stale_state.last_scan_ns = Some(99);
let meta = LocalUsageSnapshotMeta {
disk_id: "disk-test".to_string(),
..Default::default()
};
let DiskScanResult {
snapshot, state, status, ..
} = scan_disk_blocking(root.to_path_buf(), meta, stale_state).expect("scan succeeds");
assert!(state.objects.is_empty(), "stale entries should be cleared when files disappear");
assert!(
snapshot.buckets_usage.is_empty(),
"no real xl.meta files means bucket usage should stay empty"
);
assert!(status.snapshot_exists, "snapshot status should indicate a refresh");
}
#[test]
fn scan_disk_blocking_handles_large_volume() {
const OBJECTS: usize = 256;
let temp_dir = TempDir::new().expect("create temp dir");
let root = temp_dir.path();
let bucket_dir = root.join("bulk");
for idx in 0..OBJECTS {
let object_dir = bucket_dir.join(format!("obj-{idx:03}"));
fs::create_dir_all(&object_dir).expect("create object directory");
let size = 1024 + idx as i64;
let file_meta = build_file_meta_with_object(0, size);
let bytes = file_meta.marshal_msg().expect("serialize file meta");
fs::write(object_dir.join("xl.meta"), bytes).expect("write xl.meta");
}
let meta = LocalUsageSnapshotMeta {
disk_id: "disk-test".to_string(),
..Default::default()
};
let DiskScanResult { snapshot, state, .. } =
scan_disk_blocking(root.to_path_buf(), meta, IncrementalScanState::default()).expect("bulk scan succeeds");
let bucket_usage = snapshot
.buckets_usage
.get("bulk")
.expect("bucket usage present for bulk scan");
assert_eq!(bucket_usage.objects_count as usize, OBJECTS, "should count all objects once");
assert!(
bucket_usage.size >= (1024 * OBJECTS) as u64,
"aggregated size should grow with object count"
);
assert_eq!(state.objects.len(), OBJECTS, "incremental state tracks every object");
}
}
+430
View File
@@ -0,0 +1,430 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::scanner::node_scanner::{BucketStats, DiskStats, LocalScanStats};
use crate::{Error, Result};
use rustfs_common::data_usage::DataUsageInfo;
use serde::{Deserialize, Serialize};
use std::{
path::{Path, PathBuf},
sync::Arc,
sync::atomic::{AtomicU64, Ordering},
time::{Duration, SystemTime},
};
use tokio::sync::RwLock;
use tracing::{debug, error, info, warn};
/// local stats manager
pub struct LocalStatsManager {
/// node id
node_id: String,
/// stats file path
stats_file: PathBuf,
/// backup file path
backup_file: PathBuf,
/// temp file path
temp_file: PathBuf,
/// local stats data
stats: Arc<RwLock<LocalScanStats>>,
/// save interval
save_interval: Duration,
/// last save time
last_save: Arc<RwLock<SystemTime>>,
/// stats counters
counters: Arc<StatsCounters>,
}
/// stats counters
pub struct StatsCounters {
/// total scanned objects
pub total_objects_scanned: AtomicU64,
/// total healthy objects
pub total_healthy_objects: AtomicU64,
/// total corrupted objects
pub total_corrupted_objects: AtomicU64,
/// total scanned bytes
pub total_bytes_scanned: AtomicU64,
/// total scan errors
pub total_scan_errors: AtomicU64,
/// total heal triggered
pub total_heal_triggered: AtomicU64,
}
impl Default for StatsCounters {
fn default() -> Self {
Self {
total_objects_scanned: AtomicU64::new(0),
total_healthy_objects: AtomicU64::new(0),
total_corrupted_objects: AtomicU64::new(0),
total_bytes_scanned: AtomicU64::new(0),
total_scan_errors: AtomicU64::new(0),
total_heal_triggered: AtomicU64::new(0),
}
}
}
/// scan result entry
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScanResultEntry {
/// object path
pub object_path: String,
/// bucket name
pub bucket_name: String,
/// object size
pub object_size: u64,
/// is healthy
pub is_healthy: bool,
/// error message (if any)
pub error_message: Option<String>,
/// scan time
pub scan_time: SystemTime,
/// disk id
pub disk_id: String,
}
/// batch scan result
#[derive(Debug, Clone)]
pub struct BatchScanResult {
/// disk id
pub disk_id: String,
/// scan result entries
pub entries: Vec<ScanResultEntry>,
/// scan start time
pub scan_start: SystemTime,
/// scan end time
pub scan_end: SystemTime,
/// scan duration
pub scan_duration: Duration,
}
impl LocalStatsManager {
/// create new local stats manager
pub fn new(node_id: &str, data_dir: &Path) -> Self {
// ensure data directory exists
if !data_dir.exists() {
if let Err(e) = std::fs::create_dir_all(data_dir) {
error!("create stats data directory failed {:?}: {}", data_dir, e);
}
}
let stats_file = data_dir.join(format!("scanner_stats_{node_id}.json"));
let backup_file = data_dir.join(format!("scanner_stats_{node_id}.backup"));
let temp_file = data_dir.join(format!("scanner_stats_{node_id}.tmp"));
Self {
node_id: node_id.to_string(),
stats_file,
backup_file,
temp_file,
stats: Arc::new(RwLock::new(LocalScanStats::default())),
save_interval: Duration::from_secs(60), // 60 seconds save once
last_save: Arc::new(RwLock::new(SystemTime::UNIX_EPOCH)),
counters: Arc::new(StatsCounters::default()),
}
}
/// load local stats data
pub async fn load_stats(&self) -> Result<()> {
if !self.stats_file.exists() {
info!("stats data file not exists, will create new stats data");
return Ok(());
}
match self.load_stats_from_file(&self.stats_file).await {
Ok(stats) => {
*self.stats.write().await = stats;
info!("success load local stats data");
Ok(())
}
Err(e) => {
warn!("load main stats file failed: {}, try backup file", e);
match self.load_stats_from_file(&self.backup_file).await {
Ok(stats) => {
*self.stats.write().await = stats;
warn!("restore stats data from backup file");
Ok(())
}
Err(backup_e) => {
warn!("backup file also cannot load: {}, will use default stats data", backup_e);
Ok(())
}
}
}
}
}
/// load stats data from file
async fn load_stats_from_file(&self, file_path: &Path) -> Result<LocalScanStats> {
let content = tokio::fs::read_to_string(file_path)
.await
.map_err(|e| Error::IO(format!("read stats file failed: {e}")))?;
let stats: LocalScanStats =
serde_json::from_str(&content).map_err(|e| Error::Serialization(format!("deserialize stats data failed: {e}")))?;
Ok(stats)
}
/// save stats data to disk
pub async fn save_stats(&self) -> Result<()> {
let now = SystemTime::now();
let last_save = *self.last_save.read().await;
// frequency control
if now.duration_since(last_save).unwrap_or(Duration::ZERO) < self.save_interval {
return Ok(());
}
let stats = self.stats.read().await.clone();
// serialize
let json_data = serde_json::to_string_pretty(&stats)
.map_err(|e| Error::Serialization(format!("serialize stats data failed: {e}")))?;
// atomic write
tokio::fs::write(&self.temp_file, json_data)
.await
.map_err(|e| Error::IO(format!("write temp stats file failed: {e}")))?;
// backup existing file
if self.stats_file.exists() {
tokio::fs::copy(&self.stats_file, &self.backup_file)
.await
.map_err(|e| Error::IO(format!("backup stats file failed: {e}")))?;
}
// atomic replace
tokio::fs::rename(&self.temp_file, &self.stats_file)
.await
.map_err(|e| Error::IO(format!("replace stats file failed: {e}")))?;
*self.last_save.write().await = now;
debug!("save local stats data to {:?}", self.stats_file);
Ok(())
}
/// force save stats data
pub async fn force_save_stats(&self) -> Result<()> {
*self.last_save.write().await = SystemTime::UNIX_EPOCH;
self.save_stats().await
}
/// update disk scan result
pub async fn update_disk_scan_result(&self, result: &BatchScanResult) -> Result<()> {
let mut stats = self.stats.write().await;
// update disk stats
let disk_stat = stats.disks_stats.entry(result.disk_id.clone()).or_insert_with(|| DiskStats {
disk_id: result.disk_id.clone(),
..Default::default()
});
let healthy_count = result.entries.iter().filter(|e| e.is_healthy).count() as u64;
let error_count = result.entries.iter().filter(|e| !e.is_healthy).count() as u64;
disk_stat.objects_scanned += result.entries.len() as u64;
disk_stat.errors_count += error_count;
disk_stat.last_scan_time = result.scan_end;
disk_stat.scan_duration = result.scan_duration;
disk_stat.scan_completed = true;
// update overall stats
stats.objects_scanned += result.entries.len() as u64;
stats.healthy_objects += healthy_count;
stats.corrupted_objects += error_count;
stats.last_update = SystemTime::now();
// update bucket stats
for entry in &result.entries {
let _bucket_stat = stats
.buckets_stats
.entry(entry.bucket_name.clone())
.or_insert_with(BucketStats::default);
// TODO: update BucketStats
}
// update atomic counters
self.counters
.total_objects_scanned
.fetch_add(result.entries.len() as u64, Ordering::Relaxed);
self.counters
.total_healthy_objects
.fetch_add(healthy_count, Ordering::Relaxed);
self.counters
.total_corrupted_objects
.fetch_add(error_count, Ordering::Relaxed);
let total_bytes: u64 = result.entries.iter().map(|e| e.object_size).sum();
self.counters.total_bytes_scanned.fetch_add(total_bytes, Ordering::Relaxed);
if error_count > 0 {
self.counters.total_scan_errors.fetch_add(error_count, Ordering::Relaxed);
}
drop(stats);
debug!(
"update disk {} scan result: objects {}, healthy {}, error {}",
result.disk_id,
result.entries.len(),
healthy_count,
error_count
);
Ok(())
}
/// record single object scan result
pub async fn record_object_scan(&self, entry: ScanResultEntry) -> Result<()> {
let result = BatchScanResult {
disk_id: entry.disk_id.clone(),
entries: vec![entry],
scan_start: SystemTime::now(),
scan_end: SystemTime::now(),
scan_duration: Duration::from_millis(0),
};
self.update_disk_scan_result(&result).await
}
/// get local stats data copy
pub async fn get_stats(&self) -> LocalScanStats {
self.stats.read().await.clone()
}
/// get real-time counters
pub fn get_counters(&self) -> Arc<StatsCounters> {
self.counters.clone()
}
/// reset stats data
pub async fn reset_stats(&self) -> Result<()> {
{
let mut stats = self.stats.write().await;
*stats = LocalScanStats::default();
}
// reset counters
self.counters.total_objects_scanned.store(0, Ordering::Relaxed);
self.counters.total_healthy_objects.store(0, Ordering::Relaxed);
self.counters.total_corrupted_objects.store(0, Ordering::Relaxed);
self.counters.total_bytes_scanned.store(0, Ordering::Relaxed);
self.counters.total_scan_errors.store(0, Ordering::Relaxed);
self.counters.total_heal_triggered.store(0, Ordering::Relaxed);
info!("reset local stats data");
Ok(())
}
/// get stats summary
pub async fn get_stats_summary(&self) -> StatsSummary {
let stats = self.stats.read().await;
StatsSummary {
node_id: self.node_id.clone(),
total_objects_scanned: self.counters.total_objects_scanned.load(Ordering::Relaxed),
total_healthy_objects: self.counters.total_healthy_objects.load(Ordering::Relaxed),
total_corrupted_objects: self.counters.total_corrupted_objects.load(Ordering::Relaxed),
total_bytes_scanned: self.counters.total_bytes_scanned.load(Ordering::Relaxed),
total_scan_errors: self.counters.total_scan_errors.load(Ordering::Relaxed),
total_heal_triggered: self.counters.total_heal_triggered.load(Ordering::Relaxed),
total_disks: stats.disks_stats.len(),
total_buckets: stats.buckets_stats.len(),
last_update: stats.last_update,
scan_progress: stats.scan_progress.clone(),
data_usage: stats.data_usage.clone(),
}
}
/// record heal triggered
pub async fn record_heal_triggered(&self, object_path: &str, error_message: &str) {
self.counters.total_heal_triggered.fetch_add(1, Ordering::Relaxed);
info!("record heal triggered: object={}, error={}", object_path, error_message);
}
/// update data usage stats
pub async fn update_data_usage(&self, data_usage: DataUsageInfo) {
let mut stats = self.stats.write().await;
stats.data_usage = data_usage;
stats.last_update = SystemTime::now();
debug!("update data usage stats");
}
/// cleanup stats files
pub async fn cleanup_stats_files(&self) -> Result<()> {
// delete main file
if self.stats_file.exists() {
tokio::fs::remove_file(&self.stats_file)
.await
.map_err(|e| Error::IO(format!("delete stats file failed: {e}")))?;
}
// delete backup file
if self.backup_file.exists() {
tokio::fs::remove_file(&self.backup_file)
.await
.map_err(|e| Error::IO(format!("delete backup stats file failed: {e}")))?;
}
// delete temp file
if self.temp_file.exists() {
tokio::fs::remove_file(&self.temp_file)
.await
.map_err(|e| Error::IO(format!("delete temp stats file failed: {e}")))?;
}
info!("cleanup all stats files");
Ok(())
}
/// set save interval
pub fn set_save_interval(&mut self, interval: Duration) {
self.save_interval = interval;
info!("set stats data save interval to {:?}", interval);
}
}
/// stats summary
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StatsSummary {
/// node id
pub node_id: String,
/// total scanned objects
pub total_objects_scanned: u64,
/// total healthy objects
pub total_healthy_objects: u64,
/// total corrupted objects
pub total_corrupted_objects: u64,
/// total scanned bytes
pub total_bytes_scanned: u64,
/// total scan errors
pub total_scan_errors: u64,
/// total heal triggered
pub total_heal_triggered: u64,
/// total disks
pub total_disks: usize,
/// total buckets
pub total_buckets: usize,
/// last update time
pub last_update: SystemTime,
/// scan progress
pub scan_progress: super::node_scanner::ScanProgress,
/// data usage snapshot for the node
pub data_usage: DataUsageInfo,
}
+305
View File
@@ -0,0 +1,305 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
sync::atomic::{AtomicU64, Ordering},
time::{Duration, SystemTime},
};
use tracing::info;
/// Scanner metrics
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ScannerMetrics {
/// Total objects scanned since server start
pub objects_scanned: u64,
/// Total object versions scanned since server start
pub versions_scanned: u64,
/// Total directories scanned since server start
pub directories_scanned: u64,
/// Total bucket scans started since server start
pub bucket_scans_started: u64,
/// Total bucket scans finished since server start
pub bucket_scans_finished: u64,
/// Total objects with health issues found
pub objects_with_issues: u64,
/// Total heal tasks queued
pub heal_tasks_queued: u64,
/// Total heal tasks completed
pub heal_tasks_completed: u64,
/// Total heal tasks failed
pub heal_tasks_failed: u64,
/// Total healthy objects found
pub healthy_objects: u64,
/// Total corrupted objects found
pub corrupted_objects: u64,
/// Last scan activity time
pub last_activity: Option<SystemTime>,
/// Current scan cycle
pub current_cycle: u64,
/// Total scan cycles completed
pub total_cycles: u64,
/// Current scan duration
pub current_scan_duration: Option<Duration>,
/// Average scan duration
pub avg_scan_duration: Duration,
/// Objects scanned per second
pub objects_per_second: f64,
/// Buckets scanned per second
pub buckets_per_second: f64,
/// Storage metrics by bucket
pub bucket_metrics: HashMap<String, BucketMetrics>,
/// Disk metrics
pub disk_metrics: HashMap<String, DiskMetrics>,
}
/// Bucket-specific metrics
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct BucketMetrics {
/// Bucket name
pub bucket: String,
/// Total objects in bucket
pub total_objects: u64,
/// Total size of objects in bucket (bytes)
pub total_size: u64,
/// Objects with health issues
pub objects_with_issues: u64,
/// Last scan time
pub last_scan_time: Option<SystemTime>,
/// Scan duration
pub scan_duration: Option<Duration>,
/// Heal tasks queued for this bucket
pub heal_tasks_queued: u64,
/// Heal tasks completed for this bucket
pub heal_tasks_completed: u64,
/// Heal tasks failed for this bucket
pub heal_tasks_failed: u64,
}
/// Disk-specific metrics
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DiskMetrics {
/// Disk path
pub disk_path: String,
/// Total disk space (bytes)
pub total_space: u64,
/// Used disk space (bytes)
pub used_space: u64,
/// Free disk space (bytes)
pub free_space: u64,
/// Objects scanned on this disk
pub objects_scanned: u64,
/// Objects with issues on this disk
pub objects_with_issues: u64,
/// Last scan time
pub last_scan_time: Option<SystemTime>,
/// Whether disk is online
pub is_online: bool,
/// Whether disk is being scanned
pub is_scanning: bool,
}
/// Thread-safe metrics collector
pub struct MetricsCollector {
/// Atomic counters for real-time metrics
objects_scanned: AtomicU64,
versions_scanned: AtomicU64,
directories_scanned: AtomicU64,
bucket_scans_started: AtomicU64,
bucket_scans_finished: AtomicU64,
objects_with_issues: AtomicU64,
heal_tasks_queued: AtomicU64,
heal_tasks_completed: AtomicU64,
heal_tasks_failed: AtomicU64,
current_cycle: AtomicU64,
total_cycles: AtomicU64,
healthy_objects: AtomicU64,
corrupted_objects: AtomicU64,
}
impl MetricsCollector {
/// Create a new metrics collector
pub fn new() -> Self {
Self {
objects_scanned: AtomicU64::new(0),
versions_scanned: AtomicU64::new(0),
directories_scanned: AtomicU64::new(0),
bucket_scans_started: AtomicU64::new(0),
bucket_scans_finished: AtomicU64::new(0),
objects_with_issues: AtomicU64::new(0),
heal_tasks_queued: AtomicU64::new(0),
heal_tasks_completed: AtomicU64::new(0),
heal_tasks_failed: AtomicU64::new(0),
current_cycle: AtomicU64::new(0),
total_cycles: AtomicU64::new(0),
healthy_objects: AtomicU64::new(0),
corrupted_objects: AtomicU64::new(0),
}
}
/// Increment objects scanned count
pub fn increment_objects_scanned(&self, count: u64) {
self.objects_scanned.fetch_add(count, Ordering::Relaxed);
}
/// Increment versions scanned count
pub fn increment_versions_scanned(&self, count: u64) {
self.versions_scanned.fetch_add(count, Ordering::Relaxed);
}
/// Increment directories scanned count
pub fn increment_directories_scanned(&self, count: u64) {
self.directories_scanned.fetch_add(count, Ordering::Relaxed);
}
/// Increment bucket scans started count
pub fn increment_bucket_scans_started(&self, count: u64) {
self.bucket_scans_started.fetch_add(count, Ordering::Relaxed);
}
/// Increment bucket scans finished count
pub fn increment_bucket_scans_finished(&self, count: u64) {
self.bucket_scans_finished.fetch_add(count, Ordering::Relaxed);
}
/// Increment objects with issues count
pub fn increment_objects_with_issues(&self, count: u64) {
self.objects_with_issues.fetch_add(count, Ordering::Relaxed);
}
/// Increment heal tasks queued count
pub fn increment_heal_tasks_queued(&self, count: u64) {
self.heal_tasks_queued.fetch_add(count, Ordering::Relaxed);
}
/// Increment heal tasks completed count
pub fn increment_heal_tasks_completed(&self, count: u64) {
self.heal_tasks_completed.fetch_add(count, Ordering::Relaxed);
}
/// Increment heal tasks failed count
pub fn increment_heal_tasks_failed(&self, count: u64) {
self.heal_tasks_failed.fetch_add(count, Ordering::Relaxed);
}
/// Set current cycle
pub fn set_current_cycle(&self, cycle: u64) {
self.current_cycle.store(cycle, Ordering::Relaxed);
}
/// Increment total cycles
pub fn increment_total_cycles(&self) {
self.total_cycles.fetch_add(1, Ordering::Relaxed);
}
/// Increment healthy objects count
pub fn increment_healthy_objects(&self) {
self.healthy_objects.fetch_add(1, Ordering::Relaxed);
}
/// Increment corrupted objects count
pub fn increment_corrupted_objects(&self) {
self.corrupted_objects.fetch_add(1, Ordering::Relaxed);
}
/// Get current metrics snapshot
pub fn get_metrics(&self) -> ScannerMetrics {
ScannerMetrics {
objects_scanned: self.objects_scanned.load(Ordering::Relaxed),
versions_scanned: self.versions_scanned.load(Ordering::Relaxed),
directories_scanned: self.directories_scanned.load(Ordering::Relaxed),
bucket_scans_started: self.bucket_scans_started.load(Ordering::Relaxed),
bucket_scans_finished: self.bucket_scans_finished.load(Ordering::Relaxed),
objects_with_issues: self.objects_with_issues.load(Ordering::Relaxed),
heal_tasks_queued: self.heal_tasks_queued.load(Ordering::Relaxed),
heal_tasks_completed: self.heal_tasks_completed.load(Ordering::Relaxed),
heal_tasks_failed: self.heal_tasks_failed.load(Ordering::Relaxed),
healthy_objects: self.healthy_objects.load(Ordering::Relaxed),
corrupted_objects: self.corrupted_objects.load(Ordering::Relaxed),
last_activity: Some(SystemTime::now()),
current_cycle: self.current_cycle.load(Ordering::Relaxed),
total_cycles: self.total_cycles.load(Ordering::Relaxed),
current_scan_duration: None, // Will be set by scanner
avg_scan_duration: Duration::ZERO, // Will be calculated
objects_per_second: 0.0, // Will be calculated
buckets_per_second: 0.0, // Will be calculated
bucket_metrics: HashMap::new(), // Will be populated by scanner
disk_metrics: HashMap::new(), // Will be populated by scanner
}
}
/// Reset all metrics
pub fn reset(&self) {
self.objects_scanned.store(0, Ordering::Relaxed);
self.versions_scanned.store(0, Ordering::Relaxed);
self.directories_scanned.store(0, Ordering::Relaxed);
self.bucket_scans_started.store(0, Ordering::Relaxed);
self.bucket_scans_finished.store(0, Ordering::Relaxed);
self.objects_with_issues.store(0, Ordering::Relaxed);
self.heal_tasks_queued.store(0, Ordering::Relaxed);
self.heal_tasks_completed.store(0, Ordering::Relaxed);
self.heal_tasks_failed.store(0, Ordering::Relaxed);
self.current_cycle.store(0, Ordering::Relaxed);
self.total_cycles.store(0, Ordering::Relaxed);
self.healthy_objects.store(0, Ordering::Relaxed);
self.corrupted_objects.store(0, Ordering::Relaxed);
info!("Scanner metrics reset");
}
}
impl Default for MetricsCollector {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_metrics_collector_creation() {
let collector = MetricsCollector::new();
let metrics = collector.get_metrics();
assert_eq!(metrics.objects_scanned, 0);
assert_eq!(metrics.versions_scanned, 0);
}
#[test]
fn test_metrics_increment() {
let collector = MetricsCollector::new();
collector.increment_objects_scanned(10);
collector.increment_versions_scanned(5);
collector.increment_objects_with_issues(2);
let metrics = collector.get_metrics();
assert_eq!(metrics.objects_scanned, 10);
assert_eq!(metrics.versions_scanned, 5);
assert_eq!(metrics.objects_with_issues, 2);
}
#[test]
fn test_metrics_reset() {
let collector = MetricsCollector::new();
collector.increment_objects_scanned(10);
collector.reset();
let metrics = collector.get_metrics();
assert_eq!(metrics.objects_scanned, 0);
}
}
+36
View File
@@ -0,0 +1,36 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod checkpoint;
pub mod data_scanner;
pub mod histogram;
pub mod io_monitor;
pub mod io_throttler;
pub mod lifecycle;
pub mod local_scan;
pub mod local_stats;
pub mod metrics;
pub mod node_scanner;
pub mod stats_aggregator;
pub use checkpoint::{CheckpointData, CheckpointInfo, CheckpointManager};
pub use data_scanner::{ScanMode, Scanner, ScannerConfig, ScannerState};
pub use io_monitor::{AdvancedIOMonitor, IOMetrics, IOMonitorConfig};
pub use io_throttler::{AdvancedIOThrottler, IOThrottlerConfig, MetricsSnapshot, ResourceAllocation, ThrottleDecision};
pub use local_stats::{BatchScanResult, LocalStatsManager, ScanResultEntry, StatsSummary};
pub use metrics::{BucketMetrics, DiskMetrics, MetricsCollector, ScannerMetrics};
pub use node_scanner::{IOMonitor, IOThrottler, LoadLevel, LocalScanStats, NodeScanner, NodeScannerConfig};
pub use stats_aggregator::{
AggregatedStats, DecentralizedStatsAggregator, DecentralizedStatsAggregatorConfig, NodeClient, NodeInfo,
};
File diff suppressed because it is too large Load Diff
+769
View File
@@ -0,0 +1,769 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::scanner::{
local_stats::StatsSummary,
node_scanner::{BucketStats, LoadLevel, ScanProgress},
};
use crate::{Error, Result};
use rustfs_common::data_usage::DataUsageInfo;
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
sync::Arc,
time::{Duration, SystemTime},
};
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
/// node client config
#[derive(Debug, Clone)]
pub struct NodeClientConfig {
/// connect timeout
pub connect_timeout: Duration,
/// request timeout
pub request_timeout: Duration,
/// retry times
pub max_retries: u32,
/// retry interval
pub retry_interval: Duration,
}
impl Default for NodeClientConfig {
fn default() -> Self {
Self {
connect_timeout: Duration::from_secs(5),
request_timeout: Duration::from_secs(10),
max_retries: 3,
retry_interval: Duration::from_secs(1),
}
}
}
/// node info
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NodeInfo {
/// node id
pub node_id: String,
/// node address
pub address: String,
/// node port
pub port: u16,
/// is online
pub is_online: bool,
/// last heartbeat time
pub last_heartbeat: SystemTime,
/// node version
pub version: String,
}
/// aggregated stats
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AggregatedStats {
/// aggregation timestamp
pub aggregation_timestamp: SystemTime,
/// number of nodes participating in aggregation
pub node_count: usize,
/// number of online nodes
pub online_node_count: usize,
/// total scanned objects
pub total_objects_scanned: u64,
/// total healthy objects
pub total_healthy_objects: u64,
/// total corrupted objects
pub total_corrupted_objects: u64,
/// total scanned bytes
pub total_bytes_scanned: u64,
/// total scan errors
pub total_scan_errors: u64,
/// total heal triggered
pub total_heal_triggered: u64,
/// total disks
pub total_disks: usize,
/// total buckets
pub total_buckets: usize,
/// aggregated data usage
pub aggregated_data_usage: DataUsageInfo,
/// node summaries
pub node_summaries: HashMap<String, StatsSummary>,
/// aggregated bucket stats
pub aggregated_bucket_stats: HashMap<String, BucketStats>,
/// aggregated scan progress
pub scan_progress_summary: ScanProgressSummary,
/// load level distribution
pub load_level_distribution: HashMap<LoadLevel, usize>,
}
impl Default for AggregatedStats {
fn default() -> Self {
Self {
aggregation_timestamp: SystemTime::now(),
node_count: 0,
online_node_count: 0,
total_objects_scanned: 0,
total_healthy_objects: 0,
total_corrupted_objects: 0,
total_bytes_scanned: 0,
total_scan_errors: 0,
total_heal_triggered: 0,
total_disks: 0,
total_buckets: 0,
aggregated_data_usage: DataUsageInfo::default(),
node_summaries: HashMap::new(),
aggregated_bucket_stats: HashMap::new(),
scan_progress_summary: ScanProgressSummary::default(),
load_level_distribution: HashMap::new(),
}
}
}
/// scan progress summary
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ScanProgressSummary {
/// average current cycle
pub average_current_cycle: f64,
/// total completed disks
pub total_completed_disks: usize,
/// total completed buckets
pub total_completed_buckets: usize,
/// latest scan start time
pub earliest_scan_start: Option<SystemTime>,
/// estimated completion time
pub estimated_completion: Option<SystemTime>,
/// node progress
pub node_progress: HashMap<String, ScanProgress>,
}
/// node client
///
/// responsible for communicating with other nodes, getting stats data
pub struct NodeClient {
/// node info
node_info: NodeInfo,
/// config
config: NodeClientConfig,
/// HTTP client
http_client: reqwest::Client,
}
impl NodeClient {
/// create new node client
pub fn new(node_info: NodeInfo, config: NodeClientConfig) -> Self {
let http_client = reqwest::Client::builder()
.timeout(config.request_timeout)
.connect_timeout(config.connect_timeout)
.build()
.expect("Failed to create HTTP client");
Self {
node_info,
config,
http_client,
}
}
/// get node stats summary
pub async fn get_stats_summary(&self) -> Result<StatsSummary> {
let url = format!("http://{}:{}/internal/scanner/stats", self.node_info.address, self.node_info.port);
for attempt in 1..=self.config.max_retries {
match self.try_get_stats_summary(&url).await {
Ok(summary) => return Ok(summary),
Err(e) => {
warn!("try to get node {} stats failed: {}", self.node_info.node_id, e);
if attempt < self.config.max_retries {
tokio::time::sleep(self.config.retry_interval).await;
}
}
}
}
Err(Error::Other(format!("cannot get stats data from node {}", self.node_info.node_id)))
}
/// try to get stats summary
async fn try_get_stats_summary(&self, url: &str) -> Result<StatsSummary> {
let response = self
.http_client
.get(url)
.send()
.await
.map_err(|e| Error::Other(format!("HTTP request failed: {e}")))?;
if !response.status().is_success() {
return Err(Error::Other(format!("HTTP status error: {}", response.status())));
}
let summary = response
.json::<StatsSummary>()
.await
.map_err(|e| Error::Serialization(format!("deserialize stats data failed: {e}")))?;
Ok(summary)
}
/// check node health status
pub async fn check_health(&self) -> bool {
let url = format!("http://{}:{}/internal/health", self.node_info.address, self.node_info.port);
match self.http_client.get(&url).send().await {
Ok(response) => response.status().is_success(),
Err(_) => false,
}
}
/// get node info
pub fn get_node_info(&self) -> &NodeInfo {
&self.node_info
}
/// update node online status
pub fn update_online_status(&mut self, is_online: bool) {
self.node_info.is_online = is_online;
if is_online {
self.node_info.last_heartbeat = SystemTime::now();
}
}
}
/// decentralized stats aggregator config
#[derive(Debug, Clone)]
pub struct DecentralizedStatsAggregatorConfig {
/// aggregation interval
pub aggregation_interval: Duration,
/// cache ttl
pub cache_ttl: Duration,
/// node timeout
pub node_timeout: Duration,
/// max concurrent aggregations
pub max_concurrent_aggregations: usize,
}
impl Default for DecentralizedStatsAggregatorConfig {
fn default() -> Self {
Self {
aggregation_interval: Duration::from_secs(30), // 30 seconds to aggregate
cache_ttl: Duration::from_secs(3), // 3 seconds to cache
node_timeout: Duration::from_secs(5), // 5 seconds to node timeout
max_concurrent_aggregations: 10, // max 10 nodes to aggregate concurrently
}
}
}
/// decentralized stats aggregator
///
/// real-time aggregate stats data from all nodes, provide global view
pub struct DecentralizedStatsAggregator {
/// config
config: Arc<RwLock<DecentralizedStatsAggregatorConfig>>,
/// node clients
node_clients: Arc<RwLock<HashMap<String, Arc<NodeClient>>>>,
/// cached aggregated stats
cached_stats: Arc<RwLock<Option<AggregatedStats>>>,
/// cache timestamp
cache_timestamp: Arc<RwLock<SystemTime>>,
/// local node stats summary
local_stats_summary: Arc<RwLock<Option<StatsSummary>>>,
}
impl DecentralizedStatsAggregator {
/// create new decentralized stats aggregator
pub fn new(config: DecentralizedStatsAggregatorConfig) -> Self {
Self {
config: Arc::new(RwLock::new(config)),
node_clients: Arc::new(RwLock::new(HashMap::new())),
cached_stats: Arc::new(RwLock::new(None)),
cache_timestamp: Arc::new(RwLock::new(SystemTime::UNIX_EPOCH)),
local_stats_summary: Arc::new(RwLock::new(None)),
}
}
/// add node client
pub async fn add_node(&self, node_info: NodeInfo) {
let client_config = NodeClientConfig::default();
let client = Arc::new(NodeClient::new(node_info.clone(), client_config));
self.node_clients.write().await.insert(node_info.node_id.clone(), client);
info!("add node to aggregator: {}", node_info.node_id);
}
/// remove node client
pub async fn remove_node(&self, node_id: &str) {
self.node_clients.write().await.remove(node_id);
info!("remove node from aggregator: {}", node_id);
}
/// set local node stats summary
pub async fn set_local_stats(&self, stats: StatsSummary) {
*self.local_stats_summary.write().await = Some(stats);
}
/// get aggregated stats data (with cache)
pub async fn get_aggregated_stats(&self) -> Result<AggregatedStats> {
let config = self.config.read().await;
let cache_ttl = config.cache_ttl;
drop(config);
// check cache validity
let cache_timestamp = *self.cache_timestamp.read().await;
let now = SystemTime::now();
debug!(
"cache check: cache_timestamp={:?}, now={:?}, cache_ttl={:?}",
cache_timestamp, now, cache_ttl
);
// Check cache validity if timestamp is not initial value (UNIX_EPOCH)
if cache_timestamp != SystemTime::UNIX_EPOCH {
if let Ok(elapsed) = now.duration_since(cache_timestamp) {
if elapsed < cache_ttl {
if let Some(cached) = self.cached_stats.read().await.as_ref() {
debug!("Returning cached aggregated stats, remaining TTL: {:?}", cache_ttl - elapsed);
return Ok(cached.clone());
}
} else {
debug!("Cache expired: elapsed={:?} >= ttl={:?}", elapsed, cache_ttl);
}
}
}
// cache expired, re-aggregate
info!("cache expired, start re-aggregating stats data");
let aggregation_timestamp = now;
let aggregated = self.aggregate_stats_from_all_nodes(aggregation_timestamp).await?;
// update cache
*self.cached_stats.write().await = Some(aggregated.clone());
*self.cache_timestamp.write().await = aggregation_timestamp;
Ok(aggregated)
}
/// force refresh aggregated stats (ignore cache)
pub async fn force_refresh_aggregated_stats(&self) -> Result<AggregatedStats> {
let now = SystemTime::now();
let aggregated = self.aggregate_stats_from_all_nodes(now).await?;
// update cache
*self.cached_stats.write().await = Some(aggregated.clone());
*self.cache_timestamp.write().await = now;
Ok(aggregated)
}
/// aggregate stats data from all nodes
async fn aggregate_stats_from_all_nodes(&self, aggregation_timestamp: SystemTime) -> Result<AggregatedStats> {
let node_clients = self.node_clients.read().await;
let config = self.config.read().await;
// concurrent get stats data from all nodes
let mut tasks = Vec::new();
let semaphore = Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_aggregations));
// add local node stats
let mut node_summaries = HashMap::new();
if let Some(local_stats) = self.local_stats_summary.read().await.as_ref() {
node_summaries.insert(local_stats.node_id.clone(), local_stats.clone());
}
// get remote node stats
for (node_id, client) in node_clients.iter() {
let client = client.clone();
let semaphore = semaphore.clone();
let node_id = node_id.clone();
let task = tokio::spawn(async move {
let _permit = match semaphore.acquire().await {
Ok(permit) => permit,
Err(e) => {
warn!("Failed to acquire semaphore for node {}: {}", node_id, e);
return None;
}
};
match client.get_stats_summary().await {
Ok(summary) => {
debug!("successfully get node {} stats data", node_id);
Some((node_id, summary))
}
Err(e) => {
warn!("get node {} stats data failed: {}", node_id, e);
None
}
}
});
tasks.push(task);
}
// wait for all tasks to complete
for task in tasks {
if let Ok(Some((node_id, summary))) = task.await {
node_summaries.insert(node_id, summary);
}
}
drop(node_clients);
drop(config);
// aggregate stats data
let aggregated = self.aggregate_node_summaries(node_summaries, aggregation_timestamp).await;
info!(
"aggregate stats completed: {} nodes, {} online",
aggregated.node_count, aggregated.online_node_count
);
Ok(aggregated)
}
/// aggregate node summaries
async fn aggregate_node_summaries(
&self,
node_summaries: HashMap<String, StatsSummary>,
aggregation_timestamp: SystemTime,
) -> AggregatedStats {
let mut aggregated = AggregatedStats {
aggregation_timestamp,
node_count: node_summaries.len(),
online_node_count: node_summaries.len(), // assume all nodes with data are online
node_summaries: node_summaries.clone(),
..Default::default()
};
// aggregate numeric stats
for (node_id, summary) in &node_summaries {
aggregated.total_objects_scanned += summary.total_objects_scanned;
aggregated.total_healthy_objects += summary.total_healthy_objects;
aggregated.total_corrupted_objects += summary.total_corrupted_objects;
aggregated.total_bytes_scanned += summary.total_bytes_scanned;
aggregated.total_scan_errors += summary.total_scan_errors;
aggregated.total_heal_triggered += summary.total_heal_triggered;
aggregated.total_disks += summary.total_disks;
aggregated.total_buckets += summary.total_buckets;
aggregated.aggregated_data_usage.merge(&summary.data_usage);
// aggregate scan progress
aggregated
.scan_progress_summary
.node_progress
.insert(node_id.clone(), summary.scan_progress.clone());
aggregated.scan_progress_summary.total_completed_disks += summary.scan_progress.completed_disks.len();
aggregated.scan_progress_summary.total_completed_buckets += summary.scan_progress.completed_buckets.len();
}
// calculate average scan cycle
if !node_summaries.is_empty() {
let total_cycles: u64 = node_summaries.values().map(|s| s.scan_progress.current_cycle).sum();
aggregated.scan_progress_summary.average_current_cycle = total_cycles as f64 / node_summaries.len() as f64;
}
// find earliest scan start time
aggregated.scan_progress_summary.earliest_scan_start =
node_summaries.values().map(|s| s.scan_progress.scan_start_time).min();
// TODO: aggregate bucket stats and data usage
// here we need to implement it based on the specific BucketStats and DataUsageInfo structure
aggregated
}
/// get nodes health status
pub async fn get_nodes_health(&self) -> HashMap<String, bool> {
let node_clients = self.node_clients.read().await;
let mut health_status = HashMap::new();
// concurrent check all nodes health status
let mut tasks = Vec::new();
for (node_id, client) in node_clients.iter() {
let client = client.clone();
let node_id = node_id.clone();
let task = tokio::spawn(async move {
let is_healthy = client.check_health().await;
(node_id, is_healthy)
});
tasks.push(task);
}
// collect results
for task in tasks {
if let Ok((node_id, is_healthy)) = task.await {
health_status.insert(node_id, is_healthy);
}
}
health_status
}
/// get online nodes list
pub async fn get_online_nodes(&self) -> Vec<String> {
let health_status = self.get_nodes_health().await;
health_status
.into_iter()
.filter_map(|(node_id, is_healthy)| if is_healthy { Some(node_id) } else { None })
.collect()
}
/// clear cache
pub async fn clear_cache(&self) {
*self.cached_stats.write().await = None;
*self.cache_timestamp.write().await = SystemTime::UNIX_EPOCH;
info!("clear aggregated stats cache");
}
/// get cache status
pub async fn get_cache_status(&self) -> CacheStatus {
let cached_stats = self.cached_stats.read().await;
let cache_timestamp = *self.cache_timestamp.read().await;
let config = self.config.read().await;
let is_valid = if let Ok(elapsed) = SystemTime::now().duration_since(cache_timestamp) {
elapsed < config.cache_ttl
} else {
false
};
CacheStatus {
has_cached_data: cached_stats.is_some(),
cache_timestamp,
is_valid,
ttl: config.cache_ttl,
}
}
/// update config
pub async fn update_config(&self, new_config: DecentralizedStatsAggregatorConfig) {
*self.config.write().await = new_config;
info!("update aggregator config");
}
}
/// cache status
#[derive(Debug, Clone)]
pub struct CacheStatus {
/// has cached data
pub has_cached_data: bool,
/// cache timestamp
pub cache_timestamp: SystemTime,
/// cache is valid
pub is_valid: bool,
/// cache ttl
pub ttl: Duration,
}
#[cfg(test)]
mod tests {
use super::*;
use crate::scanner::node_scanner::{BucketScanState, ScanProgress};
use rustfs_common::data_usage::{BucketUsageInfo, DataUsageInfo};
use std::collections::{HashMap, HashSet};
use std::time::Duration;
#[tokio::test]
async fn aggregated_stats_merge_data_usage() {
let aggregator = DecentralizedStatsAggregator::new(DecentralizedStatsAggregatorConfig::default());
let mut data_usage = DataUsageInfo::default();
let bucket_usage = BucketUsageInfo {
objects_count: 5,
size: 1024,
..Default::default()
};
data_usage.buckets_usage.insert("bucket".to_string(), bucket_usage);
data_usage.objects_total_count = 5;
data_usage.objects_total_size = 1024;
let summary = StatsSummary {
node_id: "local-node".to_string(),
total_objects_scanned: 10,
total_healthy_objects: 9,
total_corrupted_objects: 1,
total_bytes_scanned: 2048,
total_scan_errors: 0,
total_heal_triggered: 0,
total_disks: 2,
total_buckets: 1,
last_update: SystemTime::now(),
scan_progress: ScanProgress::default(),
data_usage: data_usage.clone(),
};
aggregator.set_local_stats(summary).await;
// Wait briefly to ensure async cache writes settle in high-concurrency environments
tokio::time::sleep(Duration::from_millis(10)).await;
let aggregated = aggregator.get_aggregated_stats().await.expect("aggregated stats");
assert_eq!(aggregated.node_count, 1);
assert!(aggregated.node_summaries.contains_key("local-node"));
assert_eq!(aggregated.aggregated_data_usage.objects_total_count, 5);
assert_eq!(
aggregated
.aggregated_data_usage
.buckets_usage
.get("bucket")
.expect("bucket usage present")
.objects_count,
5
);
}
#[tokio::test]
async fn aggregated_stats_merge_multiple_nodes() {
let aggregator = DecentralizedStatsAggregator::new(DecentralizedStatsAggregatorConfig::default());
let mut local_usage = DataUsageInfo::default();
let local_bucket = BucketUsageInfo {
objects_count: 3,
versions_count: 3,
size: 150,
..Default::default()
};
local_usage.buckets_usage.insert("local-bucket".to_string(), local_bucket);
local_usage.calculate_totals();
local_usage.buckets_count = local_usage.buckets_usage.len() as u64;
local_usage.last_update = Some(SystemTime::now());
let local_progress = ScanProgress {
current_cycle: 1,
completed_disks: {
let mut set = std::collections::HashSet::new();
set.insert("disk-local".to_string());
set
},
completed_buckets: {
let mut map = std::collections::HashMap::new();
map.insert(
"local-bucket".to_string(),
BucketScanState {
completed: true,
last_object_key: Some("obj1".to_string()),
objects_scanned: 3,
scan_timestamp: SystemTime::now(),
},
);
map
},
..Default::default()
};
let local_summary = StatsSummary {
node_id: "node-local".to_string(),
total_objects_scanned: 30,
total_healthy_objects: 30,
total_corrupted_objects: 0,
total_bytes_scanned: 1500,
total_scan_errors: 0,
total_heal_triggered: 0,
total_disks: 1,
total_buckets: 1,
last_update: SystemTime::now(),
scan_progress: local_progress,
data_usage: local_usage.clone(),
};
let mut remote_usage = DataUsageInfo::default();
let remote_bucket = BucketUsageInfo {
objects_count: 5,
versions_count: 5,
size: 250,
..Default::default()
};
remote_usage.buckets_usage.insert("remote-bucket".to_string(), remote_bucket);
remote_usage.calculate_totals();
remote_usage.buckets_count = remote_usage.buckets_usage.len() as u64;
remote_usage.last_update = Some(SystemTime::now());
let remote_progress = ScanProgress {
current_cycle: 2,
completed_disks: {
let mut set = std::collections::HashSet::new();
set.insert("disk-remote".to_string());
set
},
completed_buckets: {
let mut map = std::collections::HashMap::new();
map.insert(
"remote-bucket".to_string(),
BucketScanState {
completed: true,
last_object_key: Some("remote-obj".to_string()),
objects_scanned: 5,
scan_timestamp: SystemTime::now(),
},
);
map
},
..Default::default()
};
let remote_summary = StatsSummary {
node_id: "node-remote".to_string(),
total_objects_scanned: 50,
total_healthy_objects: 48,
total_corrupted_objects: 2,
total_bytes_scanned: 2048,
total_scan_errors: 1,
total_heal_triggered: 1,
total_disks: 2,
total_buckets: 1,
last_update: SystemTime::now(),
scan_progress: remote_progress,
data_usage: remote_usage.clone(),
};
let node_summaries: HashMap<_, _> = [
(local_summary.node_id.clone(), local_summary.clone()),
(remote_summary.node_id.clone(), remote_summary.clone()),
]
.into_iter()
.collect();
let aggregated = aggregator.aggregate_node_summaries(node_summaries, SystemTime::now()).await;
assert_eq!(aggregated.node_count, 2);
assert_eq!(aggregated.total_objects_scanned, 80);
assert_eq!(aggregated.total_corrupted_objects, 2);
assert_eq!(aggregated.total_disks, 3);
assert!(aggregated.node_summaries.contains_key("node-local"));
assert!(aggregated.node_summaries.contains_key("node-remote"));
assert_eq!(
aggregated.aggregated_data_usage.objects_total_count,
local_usage.objects_total_count + remote_usage.objects_total_count
);
assert_eq!(
aggregated.aggregated_data_usage.objects_total_size,
local_usage.objects_total_size + remote_usage.objects_total_size
);
let mut expected_buckets: HashSet<&str> = HashSet::new();
expected_buckets.insert("local-bucket");
expected_buckets.insert("remote-bucket");
let actual_buckets: HashSet<&str> = aggregated
.aggregated_data_usage
.buckets_usage
.keys()
.map(|s| s.as_str())
.collect();
assert_eq!(expected_buckets, actual_buckets);
}
}
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_heal::heal::{
use rustfs_ahm::heal::{
event::{HealEvent, Severity},
task::{HealPriority, HealType},
utils,
@@ -73,7 +73,7 @@ fn test_heal_event_object_corruption() {
bucket: "test-bucket".to_string(),
object: "test-object".to_string(),
version_id: None,
corruption_type: rustfs_heal::heal::event::CorruptionType::DataCorruption,
corruption_type: rustfs_ahm::heal::event::CorruptionType::DataCorruption,
severity: Severity::High,
};
@@ -119,7 +119,7 @@ fn test_format_set_disk_id_from_i32_valid() {
#[test]
fn test_resume_state_timestamp_handling() {
use rustfs_heal::heal::resume::ResumeState;
use rustfs_ahm::heal::resume::ResumeState;
// Test that ResumeState creation doesn't panic even if system time is before epoch
// This is a theoretical test - in practice, system time should never be before epoch
@@ -139,7 +139,7 @@ fn test_resume_state_timestamp_handling() {
#[test]
fn test_resume_checkpoint_timestamp_handling() {
use rustfs_heal::heal::resume::ResumeCheckpoint;
use rustfs_ahm::heal::resume::ResumeCheckpoint;
// Test that ResumeCheckpoint creation doesn't panic
let checkpoint = ResumeCheckpoint::new("test-task".to_string());
@@ -162,8 +162,8 @@ fn test_path_to_str_helper() {
#[test]
fn test_heal_task_status_atomic_update() {
use rustfs_heal::heal::storage::HealStorageAPI;
use rustfs_heal::heal::task::{HealOptions, HealRequest, HealTask, HealTaskStatus};
use rustfs_ahm::heal::storage::HealStorageAPI;
use rustfs_ahm::heal::task::{HealOptions, HealRequest, HealTask, HealTaskStatus};
use std::sync::Arc;
// Mock storage for testing
@@ -174,49 +174,49 @@ fn test_heal_task_status_atomic_update() {
&self,
_bucket: &str,
_object: &str,
) -> rustfs_heal::Result<Option<rustfs_ecstore::store_api::ObjectInfo>> {
) -> rustfs_ahm::Result<Option<rustfs_ecstore::store_api::ObjectInfo>> {
Ok(None)
}
async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<Vec<u8>>> {
async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_ahm::Result<Option<Vec<u8>>> {
Ok(None)
}
async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_heal::Result<()> {
async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_ahm::Result<()> {
Ok(())
}
async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<()> {
async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_ahm::Result<()> {
Ok(())
}
async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<bool> {
async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_ahm::Result<bool> {
Ok(true)
}
async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Vec<u8>> {
async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_ahm::Result<Vec<u8>> {
Ok(vec![])
}
async fn get_disk_status(
&self,
_endpoint: &rustfs_ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
Ok(rustfs_heal::heal::storage::DiskStatus::Ok)
) -> rustfs_ahm::Result<rustfs_ahm::heal::storage::DiskStatus> {
Ok(rustfs_ahm::heal::storage::DiskStatus::Ok)
}
async fn format_disk(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> rustfs_heal::Result<()> {
async fn format_disk(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> rustfs_ahm::Result<()> {
Ok(())
}
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_ecstore::store_api::BucketInfo>> {
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_ahm::Result<Option<rustfs_ecstore::store_api::BucketInfo>> {
Ok(None)
}
async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> {
async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_ahm::Result<()> {
Ok(())
}
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<rustfs_ecstore::store_api::BucketInfo>> {
async fn list_buckets(&self) -> rustfs_ahm::Result<Vec<rustfs_ecstore::store_api::BucketInfo>> {
Ok(vec![])
}
async fn object_exists(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<bool> {
async fn object_exists(&self, _bucket: &str, _object: &str) -> rustfs_ahm::Result<bool> {
Ok(false)
}
async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<u64>> {
async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_ahm::Result<Option<u64>> {
Ok(None)
}
async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<String>> {
async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_ahm::Result<Option<String>> {
Ok(None)
}
async fn heal_object(
@@ -225,35 +225,27 @@ fn test_heal_task_status_atomic_update() {
_object: &str,
_version_id: Option<&str>,
_opts: &rustfs_common::heal_channel::HealOpts,
) -> rustfs_heal::Result<(rustfs_madmin::heal_commands::HealResultItem, Option<rustfs_heal::Error>)> {
) -> rustfs_ahm::Result<(rustfs_madmin::heal_commands::HealResultItem, Option<rustfs_ahm::Error>)> {
Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None))
}
async fn heal_bucket(
&self,
_bucket: &str,
_opts: &rustfs_common::heal_channel::HealOpts,
) -> rustfs_heal::Result<rustfs_madmin::heal_commands::HealResultItem> {
) -> rustfs_ahm::Result<rustfs_madmin::heal_commands::HealResultItem> {
Ok(rustfs_madmin::heal_commands::HealResultItem::default())
}
async fn heal_format(
&self,
_dry_run: bool,
) -> rustfs_heal::Result<(rustfs_madmin::heal_commands::HealResultItem, Option<rustfs_heal::Error>)> {
) -> rustfs_ahm::Result<(rustfs_madmin::heal_commands::HealResultItem, Option<rustfs_ahm::Error>)> {
Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None))
}
async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> rustfs_heal::Result<Vec<String>> {
async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> rustfs_ahm::Result<Vec<String>> {
Ok(vec![])
}
async fn list_objects_for_heal_page(
&self,
_bucket: &str,
_prefix: &str,
_continuation_token: Option<&str>,
) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> {
Ok((vec![], None, false))
}
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<rustfs_ecstore::disk::DiskStore> {
Err(rustfs_heal::Error::other("Not implemented in mock"))
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_ahm::Result<rustfs_ecstore::disk::DiskStore> {
Err(rustfs_ahm::Error::other("Not implemented in mock"))
}
}
@@ -12,7 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use http::HeaderMap;
use rustfs_ahm::heal::{
manager::{HealConfig, HealManager},
storage::{ECStoreHealStorage, HealStorageAPI},
task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
};
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_ecstore::{
disk::endpoint::Endpoint,
@@ -20,11 +24,6 @@ use rustfs_ecstore::{
store::ECStore,
store_api::{ObjectIO, ObjectOptions, PutObjReader, StorageAPI},
};
use rustfs_heal::heal::{
manager::{HealConfig, HealManager},
storage::{ECStoreHealStorage, HealStorageAPI},
task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
};
use serial_test::serial;
use std::{
path::PathBuf,
@@ -39,13 +38,9 @@ use walkdir::WalkDir;
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>)> = OnceLock::new();
static INIT: Once = Once::new();
pub fn init_tracing() {
fn init_tracing() {
INIT.call_once(|| {
let _ = tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
.with_thread_names(true)
.try_init();
let _ = tracing_subscriber::fmt::try_init();
});
}
@@ -59,7 +54,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
}
// create temp dir as 4 disks with unique base dir
let test_base_dir = format!("/tmp/rustfs_heal_heal_test_{}", uuid::Uuid::new_v4());
let test_base_dir = format!("/tmp/rustfs_ahm_heal_test_{}", uuid::Uuid::new_v4());
let temp_dir = std::path::PathBuf::from(&test_base_dir);
if temp_dir.exists() {
fs::remove_dir_all(&temp_dir).await.ok();
@@ -230,19 +225,6 @@ mod serial_tests {
// ─── 2️⃣ verify each part file is restored ───────
assert!(target_part.exists());
// ─── 3️⃣ verify object data integrity by actually reading it ───────
let mut reader = ecstore
.get_object_reader(bucket_name, object_name, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("Failed to get object reader after heal");
let mut downloaded_data = Vec::new();
tokio::io::copy(&mut reader, &mut downloaded_data)
.await
.expect("Failed to read healed object data");
assert_eq!(downloaded_data, test_data, "Healed object data does not match original");
info!("Heal object basic test passed");
}
@@ -326,27 +308,18 @@ mod serial_tests {
assert!(!format_path.exists(), "format.json still exists after deletion");
println!("✅ Deleted format.json on disk: {format_path:?}");
let (_result, error) = heal_storage.heal_format(false).await.expect("Failed to heal format");
assert!(error.is_none(), "Heal format returned error: {error:?}");
// Create heal manager with faster interval
let cfg = HealConfig {
heal_interval: Duration::from_secs(2),
..Default::default()
};
let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
heal_manager.start().await.unwrap();
// ─── 2️⃣ wait for format.json to be restored with polling + timeout ───────
// The minimal scanner interval is clamped to 10s in manager.rs, so we set timeout to 20s
let timeout_duration = Duration::from_secs(20);
let poll_interval = Duration::from_millis(200);
// Wait for task completion
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
let result = tokio::time::timeout(timeout_duration, async {
loop {
if format_path.exists() {
break;
}
tokio::time::sleep(poll_interval).await;
}
})
.await;
assert!(result.is_ok(), "format.json was not restored within timeout period");
// ─── 3️⃣ verify format.json is restored ───────
// ─── 2️⃣ verify format.json is restored ───────
assert!(format_path.exists(), "format.json does not exist on disk after heal");
info!("Heal format basic test passed");
@@ -383,55 +356,21 @@ mod serial_tests {
// Create heal manager with faster interval
let cfg = HealConfig {
heal_interval: Duration::from_secs(1),
heal_interval: Duration::from_secs(2),
..Default::default()
};
let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
heal_manager.start().await.unwrap();
// ─── 2️⃣ wait for format.json and part file to be restored with polling + timeout ───────
// The minimal scanner interval is clamped to 10s in manager.rs, so we set timeout to 20s
let timeout_duration = Duration::from_secs(20);
let poll_interval = Duration::from_millis(200);
// Wait for task completion
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
let result = tokio::time::timeout(timeout_duration, async {
loop {
if format_path.exists() && target_part.exists() {
break;
}
tokio::time::sleep(poll_interval).await;
}
})
.await;
assert!(result.is_ok(), "format.json or part file was not restored within timeout period");
// ─── 3️⃣ verify format.json is restored ───────
// ─── 2️⃣ verify format.json is restored ───────
assert!(format_path.exists(), "format.json does not exist on disk after heal");
// ─── 4️⃣ verify each part file is restored ───────
// ─── 3 verify each part file is restored ───────
assert!(target_part.exists());
// Verify object metadata is accessible
let obj_info = ecstore
.get_object_info(bucket_name, object_name, &ObjectOptions::default())
.await
.expect("Expected object to be readable after heal");
assert_eq!(obj_info.size as usize, test_data.len());
// Actually read the object data to verify integrity
let mut reader = ecstore
.get_object_reader(bucket_name, object_name, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("Failed to get object reader");
let mut downloaded_data = Vec::new();
tokio::io::copy(&mut reader, &mut downloaded_data)
.await
.expect("Failed to read object data");
assert_eq!(downloaded_data, test_data, "Healed object data does not match original");
info!("Heal format with data test passed");
info!("Heal format basic test passed");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+411
View File
@@ -0,0 +1,411 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ahm::scanner::{
io_throttler::MetricsSnapshot,
local_stats::StatsSummary,
node_scanner::{LoadLevel, NodeScanner, NodeScannerConfig},
stats_aggregator::{DecentralizedStatsAggregator, DecentralizedStatsAggregatorConfig, NodeInfo},
};
use scanner_optimization_tests::{PerformanceBenchmark, create_test_scanner};
use std::{sync::Arc, time::Duration};
use tempfile::TempDir;
mod scanner_optimization_tests;
#[tokio::test]
async fn test_end_to_end_scanner_lifecycle() {
let temp_dir = TempDir::new().unwrap();
let scanner = create_test_scanner(&temp_dir).await;
scanner.initialize_stats().await.expect("Failed to initialize stats");
let initial_progress = scanner.get_scan_progress().await;
assert_eq!(initial_progress.current_cycle, 0);
scanner.force_save_checkpoint().await.expect("Failed to save checkpoint");
let checkpoint_info = scanner.get_checkpoint_info().await.unwrap();
assert!(checkpoint_info.is_some());
}
#[tokio::test]
async fn test_load_balancing_and_throttling_integration() {
let temp_dir = TempDir::new().unwrap();
let scanner = create_test_scanner(&temp_dir).await;
let io_monitor = scanner.get_io_monitor();
let throttler = scanner.get_io_throttler();
// Start IO monitoring
io_monitor.start().await.expect("Failed to start IO monitor");
// Simulate load variation scenarios
let load_scenarios = vec![
(LoadLevel::Low, 10, 100, 0, 5), // (load level, latency, QPS, error rate, connections)
(LoadLevel::Medium, 30, 300, 10, 20),
(LoadLevel::High, 80, 800, 50, 50),
(LoadLevel::Critical, 200, 1200, 100, 100),
];
for (expected_level, latency, qps, error_rate, connections) in load_scenarios {
// Update business metrics
scanner.update_business_metrics(latency, qps, error_rate, connections).await;
// Wait for monitoring system response
tokio::time::sleep(Duration::from_millis(1200)).await;
// Get current load level
let current_level = io_monitor.get_business_load_level().await;
// Get throttling decision
let metrics_snapshot = MetricsSnapshot {
iops: 100 + qps / 10,
latency,
cpu_usage: std::cmp::min(50 + (qps / 20) as u8, 100),
memory_usage: 40,
};
let decision = throttler.make_throttle_decision(current_level, Some(metrics_snapshot)).await;
println!(
"Load scenario test: Expected={:?}, Actual={:?}, Should_pause={}, Delay={:?}",
expected_level, current_level, decision.should_pause, decision.suggested_delay
);
// Verify throttling effect under high load
if matches!(current_level, LoadLevel::High | LoadLevel::Critical) {
assert!(decision.suggested_delay > Duration::from_millis(1000));
}
if matches!(current_level, LoadLevel::Critical) {
assert!(decision.should_pause);
}
}
io_monitor.stop().await;
}
#[tokio::test]
async fn test_checkpoint_resume_functionality() {
let temp_dir = TempDir::new().unwrap();
// Create first scanner instance
let scanner1 = {
let config = NodeScannerConfig {
data_dir: temp_dir.path().to_path_buf(),
..Default::default()
};
NodeScanner::new("checkpoint-test-node".to_string(), config)
};
// Initialize and simulate some scan progress
scanner1.initialize_stats().await.unwrap();
// Simulate scan progress
scanner1
.update_scan_progress_for_test(3, 1, Some("checkpoint-test-key".to_string()))
.await;
// Save checkpoint
scanner1.force_save_checkpoint().await.unwrap();
// Stop first scanner
scanner1.stop().await.unwrap();
// Create second scanner instance (simulate restart)
let scanner2 = {
let config = NodeScannerConfig {
data_dir: temp_dir.path().to_path_buf(),
..Default::default()
};
NodeScanner::new("checkpoint-test-node".to_string(), config)
};
// Try to recover from checkpoint
scanner2.start_with_resume().await.unwrap();
// Verify recovered progress
let recovered_progress = scanner2.get_scan_progress().await;
assert_eq!(recovered_progress.current_cycle, 3);
assert_eq!(recovered_progress.current_disk_index, 1);
assert_eq!(recovered_progress.last_scan_key, Some("checkpoint-test-key".to_string()));
// Cleanup
scanner2.cleanup_checkpoint().await.unwrap();
}
#[tokio::test]
async fn test_distributed_stats_aggregation() {
// Create decentralized stats aggregator
let config = DecentralizedStatsAggregatorConfig {
cache_ttl: Duration::from_secs(10), // Increase cache TTL to ensure cache is valid during test
node_timeout: Duration::from_millis(500), // Reduce timeout
..Default::default()
};
let aggregator = DecentralizedStatsAggregator::new(config);
// Simulate multiple nodes (these nodes don't exist in test environment, will cause connection failures)
let node_infos = vec![
NodeInfo {
node_id: "node-1".to_string(),
address: "127.0.0.1".to_string(),
port: 9001,
is_online: true,
last_heartbeat: std::time::SystemTime::now(),
version: "1.0.0".to_string(),
},
NodeInfo {
node_id: "node-2".to_string(),
address: "127.0.0.1".to_string(),
port: 9002,
is_online: true,
last_heartbeat: std::time::SystemTime::now(),
version: "1.0.0".to_string(),
},
];
// Add nodes to aggregator
for node_info in node_infos {
aggregator.add_node(node_info).await;
}
// Set local statistics (simulate local node)
let local_stats = StatsSummary {
node_id: "local-node".to_string(),
total_objects_scanned: 1000,
total_healthy_objects: 950,
total_corrupted_objects: 50,
total_bytes_scanned: 1024 * 1024 * 100, // 100MB
total_scan_errors: 5,
total_heal_triggered: 10,
total_disks: 4,
total_buckets: 5,
last_update: std::time::SystemTime::now(),
scan_progress: Default::default(),
data_usage: rustfs_common::data_usage::DataUsageInfo::default(),
};
aggregator.set_local_stats(local_stats).await;
// Get aggregated statistics (remote nodes will fail, but local node should succeed)
let aggregated = aggregator.get_aggregated_stats().await.unwrap();
// Verify local node statistics are included
assert!(aggregated.node_summaries.contains_key("local-node"));
assert!(aggregated.total_objects_scanned >= 1000);
// Only local node data due to remote node connection failures
assert_eq!(aggregated.node_summaries.len(), 1);
// Test caching mechanism
let original_timestamp = aggregated.aggregation_timestamp;
let start_time = std::time::Instant::now();
let cached_result = aggregator.get_aggregated_stats().await.unwrap();
let cached_duration = start_time.elapsed();
// Verify cache is effective: timestamps should be the same
assert_eq!(original_timestamp, cached_result.aggregation_timestamp);
// Cached calls should be fast (relaxed to 200ms for test environment)
assert!(cached_duration < Duration::from_millis(200));
// Force refresh
let _refreshed = aggregator.force_refresh_aggregated_stats().await.unwrap();
// Clear cache
aggregator.clear_cache().await;
// Verify cache status
let cache_status = aggregator.get_cache_status().await;
assert!(!cache_status.has_cached_data);
}
#[tokio::test]
async fn test_performance_impact_measurement() {
let temp_dir = TempDir::new().unwrap();
let scanner = create_test_scanner(&temp_dir).await;
// Start performance monitoring
let io_monitor = scanner.get_io_monitor();
let _throttler = scanner.get_io_throttler();
io_monitor.start().await.unwrap();
// Baseline test: no scanner load - measure multiple times for stability
const MEASUREMENT_COUNT: usize = 5;
let mut baseline_measurements = Vec::new();
for _ in 0..MEASUREMENT_COUNT {
let duration = measure_workload(10_000, Duration::ZERO).await;
baseline_measurements.push(duration);
}
// Use median to reduce impact of outliers
baseline_measurements.sort();
let median_idx = baseline_measurements.len() / 2;
let baseline_duration = baseline_measurements[median_idx].max(Duration::from_millis(20));
// Simulate scanner activity
scanner.update_business_metrics(50, 500, 0, 25).await;
tokio::time::sleep(Duration::from_millis(200)).await;
// Performance test: with scanner load - measure multiple times for stability
let mut scanner_measurements = Vec::new();
for _ in 0..MEASUREMENT_COUNT {
let duration = measure_workload(10_000, Duration::ZERO).await;
scanner_measurements.push(duration);
}
scanner_measurements.sort();
let median_idx = scanner_measurements.len() / 2;
let with_scanner_duration = scanner_measurements[median_idx].max(baseline_duration);
// Calculate performance impact
let baseline_ns = baseline_duration.as_nanos().max(1) as f64;
let overhead_duration = with_scanner_duration.saturating_sub(baseline_duration);
let overhead_ns = overhead_duration.as_nanos() as f64;
let overhead_ms = (overhead_ns / 1_000_000.0).round() as u64;
let impact_percentage = (overhead_ns / baseline_ns) * 100.0;
let benchmark = PerformanceBenchmark {
_scanner_overhead_ms: overhead_ms,
business_impact_percentage: impact_percentage,
_throttle_effectiveness: 95.0, // Simulated value
};
println!("Performance impact measurement:");
println!(" Baseline duration: {baseline_duration:?}");
println!(" With scanner duration: {with_scanner_duration:?}");
println!(" Overhead: {overhead_ms} ms");
println!(" Impact percentage: {impact_percentage:.2}%");
println!(" Meets optimization goals: {}", benchmark.meets_optimization_goals());
// Verify optimization target (business impact < 50%)
// Note: In test environment, allow higher threshold due to system load variability
// In production, the actual impact should be much lower (< 10%)
assert!(impact_percentage < 50.0, "Performance impact too high: {impact_percentage:.2}%");
io_monitor.stop().await;
}
#[tokio::test]
async fn test_concurrent_scanner_operations() {
let temp_dir = TempDir::new().unwrap();
let scanner = Arc::new(create_test_scanner(&temp_dir).await);
scanner.initialize_stats().await.unwrap();
// Execute multiple scanner operations concurrently
let tasks = vec![
// Task 1: Periodically update business metrics
{
let scanner = scanner.clone();
tokio::spawn(async move {
for i in 0..10 {
scanner.update_business_metrics(10 + i * 5, 100 + i * 10, i, 5 + i).await;
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
},
// Task 2: Periodically save checkpoints
{
let scanner = scanner.clone();
tokio::spawn(async move {
for _i in 0..5 {
if let Err(e) = scanner.force_save_checkpoint().await {
eprintln!("Checkpoint save failed: {e}");
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
},
// Task 3: Periodically get statistics
{
let scanner = scanner.clone();
tokio::spawn(async move {
for _i in 0..8 {
let _summary = scanner.get_stats_summary().await;
let _progress = scanner.get_scan_progress().await;
tokio::time::sleep(Duration::from_millis(75)).await;
}
})
},
];
// Wait for all tasks to complete
for task in tasks {
task.await.unwrap();
}
// Verify final state
let final_stats = scanner.get_stats_summary().await;
let _final_progress = scanner.get_scan_progress().await;
assert_eq!(final_stats.node_id, "integration-test-node");
assert!(final_stats.last_update > std::time::SystemTime::UNIX_EPOCH);
// Cleanup
scanner.cleanup_checkpoint().await.unwrap();
}
// Helper function to simulate business workload
async fn simulate_business_workload(operations: usize) {
for _i in 0..operations {
// Simulate some CPU-intensive operations
let _result: u64 = (0..100).map(|x| x * x).sum();
// Small delay to simulate IO operations
if _i % 100 == 0 {
tokio::task::yield_now().await;
}
}
}
async fn measure_workload(operations: usize, extra_delay: Duration) -> Duration {
let start = std::time::Instant::now();
simulate_business_workload(operations).await;
if !extra_delay.is_zero() {
tokio::time::sleep(extra_delay).await;
}
start.elapsed()
}
#[tokio::test]
async fn test_error_recovery_and_resilience() {
let temp_dir = TempDir::new().unwrap();
let scanner = create_test_scanner(&temp_dir).await;
// Test recovery from stats initialization failure
scanner.initialize_stats().await.unwrap();
// Test recovery from checkpoint corruption
scanner.force_save_checkpoint().await.unwrap();
// Artificially corrupt checkpoint file (by writing invalid data)
let checkpoint_file = temp_dir.path().join("scanner_checkpoint_integration-test-node.json");
if checkpoint_file.exists() {
tokio::fs::write(&checkpoint_file, "invalid json data").await.unwrap();
}
// Verify system can gracefully handle corrupted checkpoint
let checkpoint_info = scanner.get_checkpoint_info().await;
// Should return error or null value, not crash
assert!(checkpoint_info.is_err() || checkpoint_info.unwrap().is_none());
// Clean up corrupted checkpoint
scanner.cleanup_checkpoint().await.unwrap();
// Verify ability to recreate valid checkpoint
scanner.force_save_checkpoint().await.unwrap();
let new_checkpoint_info = scanner.get_checkpoint_info().await.unwrap();
assert!(new_checkpoint_info.is_some());
}
+508
View File
@@ -0,0 +1,508 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use heed::byteorder::BigEndian;
use heed::types::*;
use heed::{BoxedError, BytesDecode, BytesEncode, Database, DatabaseFlags, Env, EnvOpenOptions};
use rustfs_ahm::scanner::local_scan::{self, LocalObjectRecord, LocalScanOutcome};
use rustfs_ecstore::{
disk::endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
store::ECStore,
store_api::{MakeBucketOptions, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader, StorageAPI},
};
use serial_test::serial;
use std::{
borrow::Cow,
path::PathBuf,
sync::{Arc, Once, OnceLock},
};
//use heed_traits::Comparator;
use time::OffsetDateTime;
use tokio::fs;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
use uuid::Uuid;
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
static INIT: Once = Once::new();
static _LIFECYCLE_EXPIRY_CURRENT_DAYS: i32 = 1;
static _LIFECYCLE_EXPIRY_NONCURRENT_DAYS: i32 = 1;
static _LIFECYCLE_TRANSITION_CURRENT_DAYS: i32 = 1;
static _LIFECYCLE_TRANSITION_NONCURRENT_DAYS: i32 = 1;
static GLOBAL_LMDB_ENV: OnceLock<Env> = OnceLock::new();
static GLOBAL_LMDB_DB: OnceLock<Database<I64<BigEndian>, LifecycleContentCodec>> = OnceLock::new();
fn init_tracing() {
INIT.call_once(|| {
let _ = tracing_subscriber::fmt::try_init();
});
}
/// Test helper: Create test environment with ECStore
async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
init_tracing();
// Fast path: already initialized, just clone and return
if let Some((paths, ecstore)) = GLOBAL_ENV.get() {
return (paths.clone(), ecstore.clone());
}
// create temp dir as 4 disks with unique base dir
let test_base_dir = format!("/tmp/rustfs_ahm_lifecyclecache_test_{}", uuid::Uuid::new_v4());
let temp_dir = std::path::PathBuf::from(&test_base_dir);
if temp_dir.exists() {
fs::remove_dir_all(&temp_dir).await.ok();
}
fs::create_dir_all(&temp_dir).await.unwrap();
// create 4 disk dirs
let disk_paths = vec![
temp_dir.join("disk1"),
temp_dir.join("disk2"),
temp_dir.join("disk3"),
temp_dir.join("disk4"),
];
for disk_path in &disk_paths {
fs::create_dir_all(disk_path).await.unwrap();
}
// create EndpointServerPools
let mut endpoints = Vec::new();
for (i, disk_path) in disk_paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
// set correct index
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
endpoints.push(endpoint);
}
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "test".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
// format disks (only first time)
rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
let port = 9002; // for simplicity
let server_addr: std::net::SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await
.unwrap();
// init bucket metadata system
let buckets_list = ecstore
.list_bucket(&rustfs_ecstore::store_api::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
//lmdb env
// User home directory
/*if let Ok(home_dir) = env::var("HOME").or_else(|_| env::var("USERPROFILE")) {
let mut path = PathBuf::from(home_dir);
path.push(format!(".{DEFAULT_LOG_FILENAME}"));
path.push(DEFAULT_LOG_DIR);
if ensure_directory_writable(&path) {
//return path;
}
}*/
let test_lmdb_lifecycle_dir = "/tmp/lmdb_lifecycle".to_string();
let temp_dir = std::path::PathBuf::from(&test_lmdb_lifecycle_dir);
if temp_dir.exists() {
fs::remove_dir_all(&temp_dir).await.ok();
}
fs::create_dir_all(&temp_dir).await.unwrap();
let lmdb_env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&test_lmdb_lifecycle_dir).unwrap() };
let bucket_name = format!("test-lc-cache-{}", "00000");
let mut wtxn = lmdb_env.write_txn().unwrap();
let db = match lmdb_env
.database_options()
.name(&format!("bucket_{}", bucket_name))
.types::<I64<BigEndian>, LifecycleContentCodec>()
.flags(DatabaseFlags::DUP_SORT)
//.dup_sort_comparator::<>()
.create(&mut wtxn)
{
Ok(db) => db,
Err(err) => {
panic!("lmdb error: {}", err);
}
};
let _ = wtxn.commit();
let _ = GLOBAL_LMDB_ENV.set(lmdb_env);
let _ = GLOBAL_LMDB_DB.set(db);
// Store in global once lock
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
(disk_paths, ecstore)
}
/// Test helper: Create a test bucket
#[allow(dead_code)]
async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
(**ecstore)
.make_bucket(bucket_name, &Default::default())
.await
.expect("Failed to create test bucket");
info!("Created test bucket: {}", bucket_name);
}
/// Test helper: Create a test lock bucket
async fn create_test_lock_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
(**ecstore)
.make_bucket(
bucket_name,
&MakeBucketOptions {
lock_enabled: true,
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("Failed to create test bucket");
info!("Created test bucket: {}", bucket_name);
}
/// Test helper: Upload test object
async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) {
let mut reader = PutObjReader::from_vec(data.to_vec());
let object_info = (**ecstore)
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("Failed to upload test object");
println!("object_info1: {:?}", object_info);
info!("Uploaded test object: {}/{} ({} bytes)", bucket, object, object_info.size);
}
/// Test helper: Check if object exists
async fn object_exists(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> bool {
match (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await {
Ok(info) => !info.delete_marker,
Err(_) => false,
}
}
fn ns_to_offset_datetime(ns: i128) -> Option<OffsetDateTime> {
OffsetDateTime::from_unix_timestamp_nanos(ns).ok()
}
fn convert_record_to_object_info(record: &LocalObjectRecord) -> ObjectInfo {
let usage = &record.usage;
ObjectInfo {
bucket: usage.bucket.clone(),
name: usage.object.clone(),
size: usage.total_size as i64,
delete_marker: !usage.has_live_object && usage.delete_markers_count > 0,
mod_time: usage.last_modified_ns.and_then(ns_to_offset_datetime),
..Default::default()
}
}
#[allow(dead_code)]
fn to_object_info(
bucket: &str,
object: &str,
total_size: i64,
delete_marker: bool,
mod_time: OffsetDateTime,
version_id: &str,
) -> ObjectInfo {
ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
size: total_size,
delete_marker,
mod_time: Some(mod_time),
version_id: Some(Uuid::parse_str(version_id).unwrap()),
..Default::default()
}
}
#[derive(Debug, PartialEq, Eq)]
enum LifecycleType {
ExpiryCurrent,
ExpiryNoncurrent,
TransitionCurrent,
TransitionNoncurrent,
}
#[derive(Debug, PartialEq, Eq)]
pub struct LifecycleContent {
ver_no: u8,
ver_id: String,
mod_time: OffsetDateTime,
type_: LifecycleType,
object_name: String,
}
pub struct LifecycleContentCodec;
impl BytesEncode<'_> for LifecycleContentCodec {
type EItem = LifecycleContent;
fn bytes_encode(lcc: &Self::EItem) -> Result<Cow<'_, [u8]>, BoxedError> {
let (ver_no_byte, ver_id_bytes, mod_timestamp_bytes, type_byte, object_name_bytes) = match lcc {
LifecycleContent {
ver_no,
ver_id,
mod_time,
type_: LifecycleType::ExpiryCurrent,
object_name,
} => (
ver_no,
ver_id.clone().into_bytes(),
mod_time.unix_timestamp().to_be_bytes(),
0,
object_name.clone().into_bytes(),
),
LifecycleContent {
ver_no,
ver_id,
mod_time,
type_: LifecycleType::ExpiryNoncurrent,
object_name,
} => (
ver_no,
ver_id.clone().into_bytes(),
mod_time.unix_timestamp().to_be_bytes(),
1,
object_name.clone().into_bytes(),
),
LifecycleContent {
ver_no,
ver_id,
mod_time,
type_: LifecycleType::TransitionCurrent,
object_name,
} => (
ver_no,
ver_id.clone().into_bytes(),
mod_time.unix_timestamp().to_be_bytes(),
2,
object_name.clone().into_bytes(),
),
LifecycleContent {
ver_no,
ver_id,
mod_time,
type_: LifecycleType::TransitionNoncurrent,
object_name,
} => (
ver_no,
ver_id.clone().into_bytes(),
mod_time.unix_timestamp().to_be_bytes(),
3,
object_name.clone().into_bytes(),
),
};
let mut output = Vec::<u8>::new();
output.push(*ver_no_byte);
output.extend_from_slice(&ver_id_bytes);
output.extend_from_slice(&mod_timestamp_bytes);
output.push(type_byte);
output.extend_from_slice(&object_name_bytes);
Ok(Cow::Owned(output))
}
}
impl<'a> BytesDecode<'a> for LifecycleContentCodec {
type DItem = LifecycleContent;
fn bytes_decode(bytes: &'a [u8]) -> Result<Self::DItem, BoxedError> {
use std::mem::size_of;
let ver_no = match bytes.get(..size_of::<u8>()) {
Some(bytes) => bytes.try_into().map(u8::from_be_bytes).unwrap(),
None => return Err("invalid LifecycleContent: cannot extract ver_no".into()),
};
let ver_id = match bytes.get(size_of::<u8>()..(36 + 1)) {
Some(bytes) => unsafe { std::str::from_utf8_unchecked(bytes).to_string() },
None => return Err("invalid LifecycleContent: cannot extract ver_id".into()),
};
let mod_timestamp = match bytes.get((36 + 1)..(size_of::<i64>() + 36 + 1)) {
Some(bytes) => bytes.try_into().map(i64::from_be_bytes).unwrap(),
None => return Err("invalid LifecycleContent: cannot extract mod_time timestamp".into()),
};
let type_ = match bytes.get(size_of::<i64>() + 36 + 1) {
Some(&0) => LifecycleType::ExpiryCurrent,
Some(&1) => LifecycleType::ExpiryNoncurrent,
Some(&2) => LifecycleType::TransitionCurrent,
Some(&3) => LifecycleType::TransitionNoncurrent,
Some(_) => return Err("invalid LifecycleContent: invalid LifecycleType".into()),
None => return Err("invalid LifecycleContent: cannot extract LifecycleType".into()),
};
let object_name = match bytes.get((size_of::<i64>() + 36 + 1 + 1)..) {
Some(bytes) => unsafe { std::str::from_utf8_unchecked(bytes).to_string() },
None => return Err("invalid LifecycleContent: cannot extract object_name".into()),
};
Ok(LifecycleContent {
ver_no,
ver_id,
mod_time: OffsetDateTime::from_unix_timestamp(mod_timestamp).unwrap(),
type_,
object_name,
})
}
}
mod serial_tests {
use super::*;
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
//#[ignore]
async fn test_lifecycle_chche_build() {
let (_disk_paths, ecstore) = setup_test_env().await;
// Create test bucket and object
let suffix = uuid::Uuid::new_v4().simple().to_string();
let bucket_name = format!("test-lc-cache-{}", &suffix[..8]);
let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
let test_data = b"Hello, this is test data for lifecycle expiry!";
create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
upload_test_object(&ecstore, bucket_name.as_str(), object_name, test_data).await;
// Verify object exists initially
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
println!("✅ Object exists before lifecycle processing");
let scan_outcome = match local_scan::scan_and_persist_local_usage(ecstore.clone()).await {
Ok(outcome) => outcome,
Err(err) => {
warn!("Local usage scan failed: {}", err);
LocalScanOutcome::default()
}
};
let bucket_objects_map = &scan_outcome.bucket_objects;
let records = match bucket_objects_map.get(&bucket_name) {
Some(records) => records,
None => {
debug!("No local snapshot entries found for bucket {}; skipping lifecycle/integrity", bucket_name);
&vec![]
}
};
if let Some(lmdb_env) = GLOBAL_LMDB_ENV.get() {
if let Some(lmdb) = GLOBAL_LMDB_DB.get() {
let mut wtxn = lmdb_env.write_txn().unwrap();
/*if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
if let Ok(object_info) = ecstore
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
.await
{
let event = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::eval_action_from_lifecycle(
&lc_config,
None,
None,
&object_info,
)
.await;
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
ecstore.clone(),
&object_info,
&event,
&rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
)
.await;
expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await;
}
}*/
for record in records {
if !record.usage.has_live_object {
continue;
}
let object_info = convert_record_to_object_info(record);
println!("object_info2: {:?}", object_info);
let mod_time = object_info.mod_time.unwrap_or(OffsetDateTime::now_utc());
let expiry_time = rustfs_ecstore::bucket::lifecycle::lifecycle::expected_expiry_time(mod_time, 1);
let version_id = if let Some(version_id) = object_info.version_id {
version_id.to_string()
} else {
"zzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz".to_string()
};
lmdb.put(
&mut wtxn,
&expiry_time.unix_timestamp(),
&LifecycleContent {
ver_no: 0,
ver_id: version_id,
mod_time,
type_: LifecycleType::TransitionNoncurrent,
object_name: object_info.name,
},
)
.unwrap();
}
wtxn.commit().unwrap();
let mut wtxn = lmdb_env.write_txn().unwrap();
let iter = lmdb.iter_mut(&mut wtxn).unwrap();
//let _ = unsafe { iter.del_current().unwrap() };
for row in iter {
if let Ok(ref elm) = row {
let LifecycleContent {
ver_no,
ver_id,
mod_time,
type_,
object_name,
} = &elm.1;
println!("cache row:{} {} {} {:?} {}", ver_no, ver_id, mod_time, type_, object_name);
}
println!("row:{:?}", row);
}
//drop(iter);
wtxn.commit().unwrap();
}
}
println!("Lifecycle cache test completed");
}
}
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ahm::scanner::{Scanner, data_scanner::ScannerConfig};
use rustfs_ecstore::{
bucket::metadata::BUCKET_LIFECYCLE_CONFIG,
bucket::metadata_sys,
@@ -22,7 +23,6 @@ use rustfs_ecstore::{
store_api::{MakeBucketOptions, ObjectIO, ObjectOptions, PutObjReader, StorageAPI},
tier::tier_config::{TierConfig, TierMinIO, TierType},
};
use rustfs_scanner::scanner::init_data_scanner;
use serial_test::serial;
use std::{
path::PathBuf,
@@ -52,7 +52,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
}
// create temp dir as 4 disks with unique base dir
let test_base_dir = format!("/tmp/rustfs_scanner_lifecycle_test_{}", uuid::Uuid::new_v4());
let test_base_dir = format!("/tmp/rustfs_ahm_lifecycle_test_{}", uuid::Uuid::new_v4());
let temp_dir = std::path::PathBuf::from(&test_base_dir);
if temp_dir.exists() {
fs::remove_dir_all(&temp_dir).await.ok();
@@ -124,7 +124,6 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
}
/// Test helper: Create a test bucket
#[allow(dead_code)]
async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
(**ecstore)
.make_bucket(bucket_name, &Default::default())
@@ -161,7 +160,6 @@ async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
}
/// Test helper: Set bucket lifecycle configuration
#[allow(dead_code)]
async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box<dyn std::error::Error>> {
// Create a simple lifecycle configuration XML with 0 days expiry for immediate testing
let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -184,7 +182,6 @@ async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box<dyn std::erro
}
/// Test helper: Set bucket lifecycle configuration
#[allow(dead_code)]
async fn set_bucket_lifecycle_deletemarker(bucket_name: &str) -> Result<(), Box<dyn std::error::Error>> {
// Create a simple lifecycle configuration XML with 0 days expiry for immediate testing
let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -261,18 +258,7 @@ async fn create_test_tier(server: u32) {
access_key: "minioadmin".to_string(),
secret_key: "minioadmin".to_string(),
bucket: "hello".to_string(),
endpoint: "http://127.0.0.1:9000".to_string(),
prefix: format!("mypre{}/", uuid::Uuid::new_v4()),
region: "".to_string(),
..Default::default()
})
} else if server == 2 {
let test_minio_server = std::env::var("TEST_MINIO_SERVER").unwrap_or_else(|_| "localhost:9000".to_string());
Some(TierMinIO {
access_key: "minioadmin".to_string(),
secret_key: "minioadmin".to_string(),
bucket: "mblock2".to_string(),
endpoint: format!("http://{}", test_minio_server),
endpoint: "http://39.105.198.204:9000".to_string(),
prefix: format!("mypre{}/", uuid::Uuid::new_v4()),
region: "".to_string(),
..Default::default()
@@ -291,11 +277,11 @@ async fn create_test_tier(server: u32) {
};
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
if let Err(err) = tier_config_mgr.add(args, false).await {
println!("tier_config_mgr add failed, e: {err:?}");
println!("tier_config_mgr add failed, e: {:?}", err);
panic!("tier add failed. {err}");
}
if let Err(e) = tier_config_mgr.save().await {
println!("tier_config_mgr save failed, e: {e:?}");
println!("tier_config_mgr save failed, e: {:?}", e);
panic!("tier save failed");
}
println!("Created test tier: COLDTIER44");
@@ -313,7 +299,7 @@ async fn object_exists(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> bo
#[allow(dead_code)]
async fn object_is_delete_marker(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> bool {
if let Ok(oi) = (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await {
println!("oi: {oi:?}");
println!("oi: {:?}", oi);
oi.delete_marker
} else {
println!("object_is_delete_marker is error");
@@ -325,7 +311,7 @@ async fn object_is_delete_marker(ecstore: &Arc<ECStore>, bucket: &str, object: &
#[allow(dead_code)]
async fn object_is_transitioned(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> bool {
if let Ok(oi) = (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await {
println!("oi: {oi:?}");
println!("oi: {:?}", oi);
!oi.transitioned_object.status.is_empty()
} else {
println!("object_is_transitioned is error");
@@ -333,7 +319,6 @@ async fn object_is_transitioned(ecstore: &Arc<ECStore>, bucket: &str, object: &s
}
}
#[allow(dead_code)]
async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
@@ -353,13 +338,271 @@ async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &
mod serial_tests {
use super::*;
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_lifecycle_expiry_basic() {
let (_disk_paths, ecstore) = setup_test_env().await;
// Create test bucket and object
let suffix = uuid::Uuid::new_v4().simple().to_string();
let bucket_name = format!("test-lc-expiry-basic-{}", &suffix[..8]);
let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
let test_data = b"Hello, this is test data for lifecycle expiry!";
create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
upload_test_object(&ecstore, bucket_name.as_str(), object_name, test_data).await;
// Verify object exists initially
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
println!("✅ Object exists before lifecycle processing");
// Set lifecycle configuration with very short expiry (0 days = immediate expiry)
set_bucket_lifecycle(bucket_name.as_str())
.await
.expect("Failed to set lifecycle configuration");
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
// Verify lifecycle configuration was set
match rustfs_ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
Ok(bucket_meta) => {
assert!(bucket_meta.lifecycle_config.is_some());
println!("✅ Bucket metadata retrieved successfully");
}
Err(e) => {
println!("❌ Error retrieving bucket metadata: {e:?}");
}
}
// Create scanner with very short intervals for testing
let scanner_config = ScannerConfig {
scan_interval: Duration::from_millis(100),
deep_scan_interval: Duration::from_millis(500),
max_concurrent_scans: 1,
..Default::default()
};
let scanner = Scanner::new(Some(scanner_config), None);
// Start scanner
scanner.start().await.expect("Failed to start scanner");
println!("✅ Scanner started");
// Wait for scanner to process lifecycle rules
tokio::time::sleep(Duration::from_secs(2)).await;
// Manually trigger a scan cycle to ensure lifecycle processing
scanner.scan_cycle().await.expect("Failed to trigger scan cycle");
println!("✅ Manual scan cycle completed");
let mut expired = false;
for attempt in 0..3 {
if attempt > 0 {
scanner.scan_cycle().await.expect("Failed to trigger scan cycle on retry");
}
expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(5)).await;
if expired {
break;
}
}
println!("Object is_delete_marker after lifecycle processing: {}", !expired);
if !expired {
let pending = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
.read()
.await
.pending_tasks()
.await;
println!("Pending expiry tasks: {pending}");
if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
if let Ok(object_info) = ecstore
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
.await
{
let event = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::eval_action_from_lifecycle(
&lc_config,
None,
None,
&object_info,
)
.await;
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
ecstore.clone(),
&object_info,
&event,
&rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
)
.await;
expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await;
}
}
if !expired {
println!("❌ Object was not deleted by lifecycle processing");
}
} else {
println!("✅ Object was successfully deleted by lifecycle processing");
// Let's try to get object info to see its details
match ecstore
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
.await
{
Ok(obj_info) => {
println!(
"Object info: name={}, size={}, mod_time={:?}",
obj_info.name, obj_info.size, obj_info.mod_time
);
}
Err(e) => {
println!("Error getting object info: {e:?}");
}
}
}
assert!(expired);
println!("✅ Object successfully expired");
// Stop scanner
let _ = scanner.stop().await;
println!("✅ Scanner stopped");
println!("Lifecycle expiry basic test completed");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
//#[ignore]
async fn test_lifecycle_expiry_deletemarker() {
let (_disk_paths, ecstore) = setup_test_env().await;
// Create test bucket and object
let suffix = uuid::Uuid::new_v4().simple().to_string();
let bucket_name = format!("test-lc-expiry-marker-{}", &suffix[..8]);
let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
let test_data = b"Hello, this is test data for lifecycle expiry!";
create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
upload_test_object(&ecstore, bucket_name.as_str(), object_name, test_data).await;
// Verify object exists initially
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
println!("✅ Object exists before lifecycle processing");
// Set lifecycle configuration with very short expiry (0 days = immediate expiry)
set_bucket_lifecycle_deletemarker(bucket_name.as_str())
.await
.expect("Failed to set lifecycle configuration");
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
// Verify lifecycle configuration was set
match rustfs_ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
Ok(bucket_meta) => {
assert!(bucket_meta.lifecycle_config.is_some());
println!("✅ Bucket metadata retrieved successfully");
}
Err(e) => {
println!("❌ Error retrieving bucket metadata: {e:?}");
}
}
// Create scanner with very short intervals for testing
let scanner_config = ScannerConfig {
scan_interval: Duration::from_millis(100),
deep_scan_interval: Duration::from_millis(500),
max_concurrent_scans: 1,
..Default::default()
};
let scanner = Scanner::new(Some(scanner_config), None);
// Start scanner
scanner.start().await.expect("Failed to start scanner");
println!("✅ Scanner started");
// Wait for scanner to process lifecycle rules
tokio::time::sleep(Duration::from_secs(2)).await;
// Manually trigger a scan cycle to ensure lifecycle processing
scanner.scan_cycle().await.expect("Failed to trigger scan cycle");
println!("✅ Manual scan cycle completed");
let mut deleted = false;
for attempt in 0..3 {
if attempt > 0 {
scanner.scan_cycle().await.expect("Failed to trigger scan cycle on retry");
}
deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(5)).await;
if deleted {
break;
}
}
println!("Object exists after lifecycle processing: {}", !deleted);
if !deleted {
let pending = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
.read()
.await
.pending_tasks()
.await;
println!("Pending expiry tasks: {pending}");
if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
if let Ok(obj_info) = ecstore
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
.await
{
let event = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::eval_action_from_lifecycle(
&lc_config, None, None, &obj_info,
)
.await;
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
ecstore.clone(),
&obj_info,
&event,
&rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
)
.await;
deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await;
if !deleted {
println!(
"Object info: name={}, size={}, mod_time={:?}",
obj_info.name, obj_info.size, obj_info.mod_time
);
}
}
}
if !deleted {
println!("❌ Object was not deleted by lifecycle processing");
}
} else {
println!("✅ Object was successfully deleted by lifecycle processing");
}
assert!(deleted);
println!("✅ Object successfully expired");
// Stop scanner
let _ = scanner.stop().await;
println!("✅ Scanner stopped");
println!("Lifecycle expiry basic test completed");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore]
async fn test_lifecycle_transition_basic() {
let (_disk_paths, ecstore) = setup_test_env().await;
create_test_tier(2).await;
create_test_tier(1).await;
// Create test bucket and object
let suffix = uuid::Uuid::new_v4().simple().to_string();
@@ -367,15 +610,8 @@ mod serial_tests {
let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
let test_data = b"Hello, this is test data for lifecycle expiry!";
create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
upload_test_object(
&ecstore,
bucket_name.as_str(),
object_name,
b"Hello, this is test data for lifecycle expiry 1111-11111111-1111 !",
)
.await;
//create_test_bucket(&ecstore, bucket_name.as_str()).await;
//create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
create_test_bucket(&ecstore, bucket_name.as_str()).await;
upload_test_object(&ecstore, bucket_name.as_str(), object_name, test_data).await;
// Verify object exists initially
@@ -399,14 +635,29 @@ mod serial_tests {
}
}
let ctx = CancellationToken::new();
// Create scanner with very short intervals for testing
let scanner_config = ScannerConfig {
scan_interval: Duration::from_millis(100),
deep_scan_interval: Duration::from_millis(500),
max_concurrent_scans: 1,
..Default::default()
};
let scanner = Scanner::new(Some(scanner_config), None);
// Start scanner
init_data_scanner(ctx.clone(), ecstore.clone()).await;
scanner.start().await.expect("Failed to start scanner");
println!("✅ Scanner started");
// Wait for scanner to process lifecycle rules
tokio::time::sleep(Duration::from_secs(1200)).await;
tokio::time::sleep(Duration::from_secs(2)).await;
// Manually trigger a scan cycle to ensure lifecycle processing
scanner.scan_cycle().await.expect("Failed to trigger scan cycle");
println!("✅ Manual scan cycle completed");
// Wait a bit more for background workers to process expiry tasks
tokio::time::sleep(Duration::from_secs(5)).await;
// Check if object has been expired (deleted)
let check_result = object_is_transitioned(&ecstore, &bucket_name, object_name).await;
@@ -438,7 +689,7 @@ mod serial_tests {
println!("✅ Object successfully transitioned");
// Stop scanner
ctx.cancel();
let _ = scanner.stop().await;
println!("✅ Scanner stopped");
println!("Lifecycle transition basic test completed");
+817
View File
@@ -0,0 +1,817 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ahm::heal::manager::HealConfig;
use rustfs_ahm::scanner::{
Scanner,
data_scanner::ScanMode,
node_scanner::{LoadLevel, NodeScanner, NodeScannerConfig},
};
use rustfs_ecstore::{
StorageAPI,
disk::endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
store::ECStore,
store_api::{MakeBucketOptions, ObjectIO, PutObjReader},
};
use serial_test::serial;
use std::{fs, net::SocketAddr, sync::Arc, sync::OnceLock, time::Duration};
use tempfile::TempDir;
use tokio_util::sync::CancellationToken;
// Global test environment cache to avoid repeated initialization
static GLOBAL_TEST_ENV: OnceLock<(Vec<std::path::PathBuf>, Arc<ECStore>)> = OnceLock::new();
async fn prepare_test_env(test_dir: Option<&str>, port: Option<u16>) -> (Vec<std::path::PathBuf>, Arc<ECStore>) {
// Check if global environment is already initialized
if let Some((disk_paths, ecstore)) = GLOBAL_TEST_ENV.get() {
return (disk_paths.clone(), ecstore.clone());
}
// create temp dir as 4 disks
let test_base_dir = test_dir.unwrap_or("/tmp/rustfs_ahm_optimized_test");
let temp_dir = std::path::PathBuf::from(test_base_dir);
if temp_dir.exists() {
fs::remove_dir_all(&temp_dir).unwrap();
}
fs::create_dir_all(&temp_dir).unwrap();
// create 4 disk dirs
let disk_paths = vec![
temp_dir.join("disk1"),
temp_dir.join("disk2"),
temp_dir.join("disk3"),
temp_dir.join("disk4"),
];
for disk_path in &disk_paths {
fs::create_dir_all(disk_path).unwrap();
}
// create EndpointServerPools
let mut endpoints = Vec::new();
for (i, disk_path) in disk_paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
// set correct index
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
endpoints.push(endpoint);
}
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "test".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
// format disks
rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
// create ECStore with dynamic port
let port = port.unwrap_or(9000);
let server_addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await
.unwrap();
// init bucket metadata system
let buckets_list = ecstore
.list_bucket(&rustfs_ecstore::store_api::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
// Store in global cache
let _ = GLOBAL_TEST_ENV.set((disk_paths.clone(), ecstore.clone()));
(disk_paths, ecstore)
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Please run it manually."]
#[serial]
async fn test_optimized_scanner_basic_functionality() {
const TEST_DIR_BASIC: &str = "/tmp/rustfs_ahm_optimized_test_basic";
let (disk_paths, ecstore) = prepare_test_env(Some(TEST_DIR_BASIC), Some(9101)).await;
// create some test data
let bucket_name = "test-bucket";
let object_name = "test-object";
let test_data = b"Hello, Optimized RustFS!";
// create bucket and verify
let bucket_opts = MakeBucketOptions::default();
ecstore
.make_bucket(bucket_name, &bucket_opts)
.await
.expect("make_bucket failed");
// check bucket really exists
let buckets = ecstore
.list_bucket(&rustfs_ecstore::store_api::BucketOptions::default())
.await
.unwrap();
assert!(buckets.iter().any(|b| b.name == bucket_name), "bucket not found after creation");
// write object
let mut put_reader = PutObjReader::from_vec(test_data.to_vec());
let object_opts = rustfs_ecstore::store_api::ObjectOptions::default();
ecstore
.put_object(bucket_name, object_name, &mut put_reader, &object_opts)
.await
.expect("put_object failed");
// create optimized Scanner and test basic functionality
let scanner = Scanner::new(None, None);
// Test 1: Normal scan - verify object is found
println!("=== Test 1: Optimized Normal scan ===");
let scan_result = scanner.scan_cycle().await;
assert!(scan_result.is_ok(), "Optimized normal scan should succeed");
let _metrics = scanner.get_metrics().await;
// Note: The optimized scanner may not immediately show scanned objects as it works differently
println!("Optimized normal scan completed successfully");
// Test 2: Simulate disk corruption - delete object data from disk1
println!("=== Test 2: Optimized corruption handling ===");
let disk1_bucket_path = disk_paths[0].join(bucket_name);
let disk1_object_path = disk1_bucket_path.join(object_name);
// Try to delete the object file from disk1 (simulate corruption)
// Note: This might fail if ECStore is actively using the file
match fs::remove_dir_all(&disk1_object_path) {
Ok(_) => {
println!("Successfully deleted object from disk1: {disk1_object_path:?}");
// Verify deletion by checking if the directory still exists
if disk1_object_path.exists() {
println!("WARNING: Directory still exists after deletion: {disk1_object_path:?}");
} else {
println!("Confirmed: Directory was successfully deleted");
}
}
Err(e) => {
println!("Could not delete object from disk1 (file may be in use): {disk1_object_path:?} - {e}");
// This is expected behavior - ECStore might be holding file handles
}
}
// Scan again - should still complete (even with missing data)
let scan_result_after_corruption = scanner.scan_cycle().await;
println!("Optimized scan after corruption result: {scan_result_after_corruption:?}");
// Scanner should handle missing data gracefully
assert!(
scan_result_after_corruption.is_ok(),
"Optimized scanner should handle missing data gracefully"
);
// Test 3: Test metrics collection
println!("=== Test 3: Optimized metrics collection ===");
let final_metrics = scanner.get_metrics().await;
println!("Optimized final metrics: {final_metrics:?}");
// Verify metrics are available (even if different from legacy scanner)
assert!(final_metrics.last_activity.is_some(), "Should have scan activity");
// clean up temp dir
let temp_dir = std::path::PathBuf::from(TEST_DIR_BASIC);
if let Err(e) = fs::remove_dir_all(&temp_dir) {
eprintln!("Warning: Failed to clean up temp directory {temp_dir:?}: {e}");
}
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Please run it manually."]
#[serial]
async fn test_optimized_scanner_usage_stats() {
const TEST_DIR_USAGE_STATS: &str = "/tmp/rustfs_ahm_optimized_test_usage_stats";
let (_, ecstore) = prepare_test_env(Some(TEST_DIR_USAGE_STATS), Some(9102)).await;
// prepare test bucket and object
let bucket = "test-bucket-optimized";
ecstore.make_bucket(bucket, &Default::default()).await.unwrap();
let mut pr = PutObjReader::from_vec(b"hello optimized".to_vec());
ecstore
.put_object(bucket, "obj1", &mut pr, &Default::default())
.await
.unwrap();
let scanner = Scanner::new(None, None);
// enable statistics
scanner.set_config_enable_data_usage_stats(true).await;
// first scan and get statistics
scanner.scan_cycle().await.unwrap();
let du_initial = scanner.get_data_usage_info().await.unwrap();
// Note: Optimized scanner may work differently, so we're less strict about counts
println!("Initial data usage: {du_initial:?}");
// write 3 more objects and get statistics again
for size in [1024, 2048, 4096] {
let name = format!("obj_{size}");
let mut pr = PutObjReader::from_vec(vec![b'x'; size]);
ecstore.put_object(bucket, &name, &mut pr, &Default::default()).await.unwrap();
}
scanner.scan_cycle().await.unwrap();
let du_after = scanner.get_data_usage_info().await.unwrap();
println!("Data usage after adding objects: {du_after:?}");
// The optimized scanner should at least not crash and return valid data
// buckets_count is u64, so it's always >= 0
assert!(du_after.buckets_count == du_after.buckets_count);
// clean up temp dir
let _ = std::fs::remove_dir_all(std::path::Path::new(TEST_DIR_USAGE_STATS));
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Please run it manually."]
#[serial]
async fn test_optimized_volume_healing_functionality() {
const TEST_DIR_VOLUME_HEAL: &str = "/tmp/rustfs_ahm_optimized_test_volume_heal";
let (disk_paths, ecstore) = prepare_test_env(Some(TEST_DIR_VOLUME_HEAL), Some(9103)).await;
// Create test buckets
let bucket1 = "test-bucket-1-opt";
let bucket2 = "test-bucket-2-opt";
ecstore.make_bucket(bucket1, &Default::default()).await.unwrap();
ecstore.make_bucket(bucket2, &Default::default()).await.unwrap();
// Add some test objects
let mut pr1 = PutObjReader::from_vec(b"test data 1 optimized".to_vec());
ecstore
.put_object(bucket1, "obj1", &mut pr1, &Default::default())
.await
.unwrap();
let mut pr2 = PutObjReader::from_vec(b"test data 2 optimized".to_vec());
ecstore
.put_object(bucket2, "obj2", &mut pr2, &Default::default())
.await
.unwrap();
// Simulate missing bucket on one disk by removing bucket directory
let disk1_bucket1_path = disk_paths[0].join(bucket1);
if disk1_bucket1_path.exists() {
println!("Removing bucket directory to simulate missing volume: {disk1_bucket1_path:?}");
match fs::remove_dir_all(&disk1_bucket1_path) {
Ok(_) => println!("Successfully removed bucket directory from disk 0"),
Err(e) => println!("Failed to remove bucket directory: {e}"),
}
}
// Create optimized scanner
let scanner = Scanner::new(None, None);
// Enable healing in config
scanner.set_config_enable_healing(true).await;
println!("=== Testing optimized volume healing functionality ===");
// Run scan cycle which should detect missing volume
let scan_result = scanner.scan_cycle().await;
assert!(scan_result.is_ok(), "Optimized scan cycle should succeed");
// Get metrics to verify scan completed
let metrics = scanner.get_metrics().await;
println!("Optimized volume healing detection test completed successfully");
println!("Optimized scan metrics: {metrics:?}");
// Clean up
let _ = std::fs::remove_dir_all(std::path::Path::new(TEST_DIR_VOLUME_HEAL));
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Please run it manually."]
#[serial]
async fn test_optimized_performance_characteristics() {
const TEST_DIR_PERF: &str = "/tmp/rustfs_ahm_optimized_test_perf";
let (_, ecstore) = prepare_test_env(Some(TEST_DIR_PERF), Some(9104)).await;
// Create test bucket with multiple objects
let bucket_name = "performance-test-bucket";
ecstore.make_bucket(bucket_name, &Default::default()).await.unwrap();
// Create several test objects
for i in 0..10 {
let object_name = format!("perf-object-{i}");
let test_data = vec![b'A' + (i % 26) as u8; 1024 * (i + 1)]; // Variable size objects
let mut put_reader = PutObjReader::from_vec(test_data);
let object_opts = rustfs_ecstore::store_api::ObjectOptions::default();
ecstore
.put_object(bucket_name, &object_name, &mut put_reader, &object_opts)
.await
.unwrap_or_else(|_| panic!("Failed to create object {object_name}"));
}
// Create optimized scanner
let scanner = Scanner::new(None, None);
// Test performance characteristics
println!("=== Testing optimized scanner performance ===");
// Measure scan time
let start_time = std::time::Instant::now();
let scan_result = scanner.scan_cycle().await;
let scan_duration = start_time.elapsed();
println!("Optimized scan completed in: {scan_duration:?}");
assert!(scan_result.is_ok(), "Performance scan should succeed");
// Verify the scan was reasonably fast (should be faster than old concurrent scanner)
// Note: This is a rough check - in practice, optimized scanner should be much faster
assert!(
scan_duration < Duration::from_secs(30),
"Optimized scan should complete within 30 seconds"
);
// Test memory usage is reasonable (indirect test through successful completion)
let metrics = scanner.get_metrics().await;
println!("Performance test metrics: {metrics:?}");
// Test that multiple scans don't degrade performance significantly
let start_time2 = std::time::Instant::now();
let _scan_result2 = scanner.scan_cycle().await;
let scan_duration2 = start_time2.elapsed();
println!("Second optimized scan completed in: {scan_duration2:?}");
// Second scan should be similar or faster due to caching
let performance_ratio = scan_duration2.as_millis() as f64 / scan_duration.as_millis() as f64;
println!("Performance ratio (second/first): {performance_ratio:.2}");
// Clean up
let _ = std::fs::remove_dir_all(std::path::Path::new(TEST_DIR_PERF));
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Please run it manually."]
#[serial]
async fn test_optimized_load_balancing_and_throttling() {
let temp_dir = TempDir::new().unwrap();
// Create a node scanner with optimized configuration
let config = NodeScannerConfig {
data_dir: temp_dir.path().to_path_buf(),
enable_smart_scheduling: true,
scan_interval: Duration::from_millis(100), // Fast for testing
disk_scan_delay: Duration::from_millis(50),
..Default::default()
};
let node_scanner = NodeScanner::new("test-optimized-node".to_string(), config);
// Initialize the scanner
node_scanner.initialize_stats().await.unwrap();
let io_monitor = node_scanner.get_io_monitor();
let throttler = node_scanner.get_io_throttler();
// Start IO monitoring
io_monitor.start().await.expect("Failed to start IO monitor");
// Test load balancing scenarios
let load_scenarios = vec![
(LoadLevel::Low, 10, 100, 0, 5), // (load level, latency, qps, error rate, connections)
(LoadLevel::Medium, 30, 300, 10, 20),
(LoadLevel::High, 80, 800, 50, 50),
(LoadLevel::Critical, 200, 1200, 100, 100),
];
for (expected_level, latency, qps, error_rate, connections) in load_scenarios {
println!("Testing load scenario: {expected_level:?}");
// Update business metrics to simulate load
node_scanner
.update_business_metrics(latency, qps, error_rate, connections)
.await;
// Wait for monitoring system to respond
tokio::time::sleep(Duration::from_millis(500)).await;
// Get current load level
let current_level = io_monitor.get_business_load_level().await;
println!("Detected load level: {current_level:?}");
// Get throttling decision
let _current_metrics = io_monitor.get_current_metrics().await;
let metrics_snapshot = rustfs_ahm::scanner::io_throttler::MetricsSnapshot {
iops: 100 + qps / 10,
latency,
cpu_usage: std::cmp::min(50 + (qps / 20) as u8, 100),
memory_usage: 40,
};
let decision = throttler.make_throttle_decision(current_level, Some(metrics_snapshot)).await;
println!(
"Throttle decision: should_pause={}, delay={:?}",
decision.should_pause, decision.suggested_delay
);
// Verify throttling behavior
match current_level {
LoadLevel::Critical => {
assert!(decision.should_pause, "Critical load should trigger pause");
}
LoadLevel::High => {
assert!(
decision.suggested_delay > Duration::from_millis(1000),
"High load should suggest significant delay"
);
}
_ => {
// Lower loads should have reasonable delays
assert!(
decision.suggested_delay < Duration::from_secs(5),
"Lower loads should not have excessive delays"
);
}
}
}
io_monitor.stop().await;
println!("Optimized load balancing and throttling test completed successfully");
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Please run it manually."]
#[serial]
async fn test_optimized_scanner_detect_missing_data_parts() {
const TEST_DIR_MISSING_PARTS: &str = "/tmp/rustfs_ahm_optimized_test_missing_parts";
let (disk_paths, ecstore) = prepare_test_env(Some(TEST_DIR_MISSING_PARTS), Some(9105)).await;
// Create test bucket
let bucket_name = "test-bucket-parts-opt";
let object_name = "large-object-20mb-opt";
ecstore.make_bucket(bucket_name, &Default::default()).await.unwrap();
// Create a 20MB object to ensure it has multiple parts
let large_data = vec![b'A'; 20 * 1024 * 1024]; // 20MB of 'A' characters
let mut put_reader = PutObjReader::from_vec(large_data);
let object_opts = rustfs_ecstore::store_api::ObjectOptions::default();
println!("=== Creating 20MB object ===");
ecstore
.put_object(bucket_name, object_name, &mut put_reader, &object_opts)
.await
.expect("put_object failed for large object");
// Verify object was created and get its info
let obj_info = ecstore
.get_object_info(bucket_name, object_name, &object_opts)
.await
.expect("get_object_info failed");
println!(
"Object info: size={}, parts={}, inlined={}",
obj_info.size,
obj_info.parts.len(),
obj_info.inlined
);
assert!(!obj_info.inlined, "20MB object should not be inlined");
println!("Object has {} parts", obj_info.parts.len());
// Create HealManager and optimized Scanner
let heal_storage = Arc::new(rustfs_ahm::heal::storage::ECStoreHealStorage::new(ecstore.clone()));
let heal_config = HealConfig {
enable_auto_heal: true,
heal_interval: Duration::from_millis(100),
max_concurrent_heals: 4,
task_timeout: Duration::from_secs(300),
queue_size: 1000,
};
let heal_manager = Arc::new(rustfs_ahm::heal::HealManager::new(heal_storage, Some(heal_config)));
heal_manager.start().await.unwrap();
let scanner = Scanner::new(None, Some(heal_manager.clone()));
// Enable healing to detect missing parts
scanner.set_config_enable_healing(true).await;
scanner.set_config_scan_mode(ScanMode::Deep).await;
println!("=== Initial scan (all parts present) ===");
let initial_scan = scanner.scan_cycle().await;
assert!(initial_scan.is_ok(), "Initial scan should succeed");
let initial_metrics = scanner.get_metrics().await;
println!("Initial scan metrics: objects_scanned={}", initial_metrics.objects_scanned);
// Simulate data part loss by deleting part files from some disks
println!("=== Simulating data part loss ===");
let mut deleted_parts = 0;
let mut deleted_part_paths = Vec::new();
for (disk_idx, disk_path) in disk_paths.iter().enumerate() {
if disk_idx > 0 {
// Only delete from first disk
break;
}
let bucket_path = disk_path.join(bucket_name);
let object_path = bucket_path.join(object_name);
if !object_path.exists() {
continue;
}
// Find the data directory (UUID)
if let Ok(entries) = fs::read_dir(&object_path) {
for entry in entries.flatten() {
let entry_path = entry.path();
if entry_path.is_dir() {
// This is likely the data_dir, look for part files inside
let part_file_path = entry_path.join("part.1");
if part_file_path.exists() {
match fs::remove_file(&part_file_path) {
Ok(_) => {
println!("Deleted part file: {part_file_path:?}");
deleted_part_paths.push(part_file_path);
deleted_parts += 1;
}
Err(e) => {
println!("Failed to delete part file {part_file_path:?}: {e}");
}
}
}
}
}
}
}
println!("Deleted {deleted_parts} part files to simulate data loss");
// Scan again to detect missing parts
println!("=== Scan after data deletion (should detect missing data) ===");
let scan_after_deletion = scanner.scan_cycle().await;
// Wait a bit for the heal manager to process
tokio::time::sleep(Duration::from_millis(500)).await;
// Check heal statistics
let heal_stats = heal_manager.get_statistics().await;
println!("Heal statistics:");
println!(" - total_tasks: {}", heal_stats.total_tasks);
println!(" - successful_tasks: {}", heal_stats.successful_tasks);
println!(" - failed_tasks: {}", heal_stats.failed_tasks);
// Get scanner metrics
let final_metrics = scanner.get_metrics().await;
println!("Scanner metrics after deletion scan:");
println!(" - objects_scanned: {}", final_metrics.objects_scanned);
// The optimized scanner should handle missing data gracefully
match scan_after_deletion {
Ok(_) => {
println!("Optimized scanner completed successfully despite missing data");
}
Err(e) => {
println!("Optimized scanner detected errors (acceptable): {e}");
}
}
println!("=== Test completed ===");
println!("Optimized scanner successfully handled missing data scenario");
// Clean up
let _ = std::fs::remove_dir_all(std::path::Path::new(TEST_DIR_MISSING_PARTS));
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Please run it manually."]
#[serial]
async fn test_optimized_scanner_detect_missing_xl_meta() {
const TEST_DIR_MISSING_META: &str = "/tmp/rustfs_ahm_optimized_test_missing_meta";
let (disk_paths, ecstore) = prepare_test_env(Some(TEST_DIR_MISSING_META), Some(9106)).await;
// Create test bucket
let bucket_name = "test-bucket-meta-opt";
let object_name = "test-object-meta-opt";
ecstore.make_bucket(bucket_name, &Default::default()).await.unwrap();
// Create a test object
let test_data = vec![b'B'; 5 * 1024 * 1024]; // 5MB of 'B' characters
let mut put_reader = PutObjReader::from_vec(test_data);
let object_opts = rustfs_ecstore::store_api::ObjectOptions::default();
println!("=== Creating test object ===");
ecstore
.put_object(bucket_name, object_name, &mut put_reader, &object_opts)
.await
.expect("put_object failed");
// Create HealManager and optimized Scanner
let heal_storage = Arc::new(rustfs_ahm::heal::storage::ECStoreHealStorage::new(ecstore.clone()));
let heal_config = HealConfig {
enable_auto_heal: true,
heal_interval: Duration::from_millis(100),
max_concurrent_heals: 4,
task_timeout: Duration::from_secs(300),
queue_size: 1000,
};
let heal_manager = Arc::new(rustfs_ahm::heal::HealManager::new(heal_storage, Some(heal_config)));
heal_manager.start().await.unwrap();
let scanner = Scanner::new(None, Some(heal_manager.clone()));
// Enable healing to detect missing metadata
scanner.set_config_enable_healing(true).await;
scanner.set_config_scan_mode(ScanMode::Deep).await;
println!("=== Initial scan (all metadata present) ===");
let initial_scan = scanner.scan_cycle().await;
assert!(initial_scan.is_ok(), "Initial scan should succeed");
// Simulate xl.meta file loss by deleting xl.meta files from some disks
println!("=== Simulating xl.meta file loss ===");
let mut deleted_meta_files = 0;
let mut deleted_meta_paths = Vec::new();
for (disk_idx, disk_path) in disk_paths.iter().enumerate() {
if disk_idx >= 2 {
// Only delete from first two disks to ensure some copies remain
break;
}
let bucket_path = disk_path.join(bucket_name);
let object_path = bucket_path.join(object_name);
if !object_path.exists() {
continue;
}
// Delete xl.meta file
let xl_meta_path = object_path.join("xl.meta");
if xl_meta_path.exists() {
match fs::remove_file(&xl_meta_path) {
Ok(_) => {
println!("Deleted xl.meta file: {xl_meta_path:?}");
deleted_meta_paths.push(xl_meta_path);
deleted_meta_files += 1;
}
Err(e) => {
println!("Failed to delete xl.meta file {xl_meta_path:?}: {e}");
}
}
}
}
println!("Deleted {deleted_meta_files} xl.meta files to simulate metadata loss");
// Scan again to detect missing metadata
println!("=== Scan after xl.meta deletion ===");
let scan_after_deletion = scanner.scan_cycle().await;
// Wait for heal manager to process
tokio::time::sleep(Duration::from_millis(1000)).await;
// Check heal statistics
let final_heal_stats = heal_manager.get_statistics().await;
println!("Final heal statistics:");
println!(" - total_tasks: {}", final_heal_stats.total_tasks);
println!(" - successful_tasks: {}", final_heal_stats.successful_tasks);
println!(" - failed_tasks: {}", final_heal_stats.failed_tasks);
let _ = final_heal_stats; // Use the variable to avoid unused warning
// The optimized scanner should handle missing metadata gracefully
match scan_after_deletion {
Ok(_) => {
println!("Optimized scanner completed successfully despite missing metadata");
}
Err(e) => {
println!("Optimized scanner detected errors (acceptable): {e}");
}
}
println!("=== Test completed ===");
println!("Optimized scanner successfully handled missing xl.meta scenario");
// Clean up
let _ = std::fs::remove_dir_all(std::path::Path::new(TEST_DIR_MISSING_META));
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Please run it manually."]
#[serial]
async fn test_optimized_scanner_healthy_objects_not_marked_corrupted() {
const TEST_DIR_HEALTHY: &str = "/tmp/rustfs_ahm_optimized_test_healthy_objects";
let (_, ecstore) = prepare_test_env(Some(TEST_DIR_HEALTHY), Some(9107)).await;
// Create heal manager for this test
let heal_config = HealConfig::default();
let heal_storage = Arc::new(rustfs_ahm::heal::storage::ECStoreHealStorage::new(ecstore.clone()));
let heal_manager = Arc::new(rustfs_ahm::heal::manager::HealManager::new(heal_storage, Some(heal_config)));
heal_manager.start().await.unwrap();
// Create optimized scanner with healing enabled
let scanner = Scanner::new(None, Some(heal_manager.clone()));
scanner.set_config_enable_healing(true).await;
scanner.set_config_scan_mode(ScanMode::Deep).await;
// Create test bucket and multiple healthy objects
let bucket_name = "healthy-test-bucket-opt";
let bucket_opts = MakeBucketOptions::default();
ecstore.make_bucket(bucket_name, &bucket_opts).await.unwrap();
// Create multiple test objects with different sizes
let test_objects = vec![
("small-object-opt", b"Small test data optimized".to_vec()),
("medium-object-opt", vec![42u8; 1024]), // 1KB
("large-object-opt", vec![123u8; 10240]), // 10KB
];
let object_opts = rustfs_ecstore::store_api::ObjectOptions::default();
// Write all test objects
for (object_name, test_data) in &test_objects {
let mut put_reader = PutObjReader::from_vec(test_data.clone());
ecstore
.put_object(bucket_name, object_name, &mut put_reader, &object_opts)
.await
.expect("Failed to put test object");
println!("Created test object: {object_name} (size: {} bytes)", test_data.len());
}
// Wait a moment for objects to be fully written
tokio::time::sleep(Duration::from_millis(100)).await;
// Get initial heal statistics
let initial_heal_stats = heal_manager.get_statistics().await;
println!("Initial heal statistics:");
println!(" - total_tasks: {}", initial_heal_stats.total_tasks);
// Perform initial scan on healthy objects
println!("=== Scanning healthy objects ===");
let scan_result = scanner.scan_cycle().await;
assert!(scan_result.is_ok(), "Scan of healthy objects should succeed");
// Wait for any potential heal tasks to be processed
tokio::time::sleep(Duration::from_millis(1000)).await;
// Get scanner metrics after scanning
let metrics = scanner.get_metrics().await;
println!("Optimized scanner metrics after scanning healthy objects:");
println!(" - objects_scanned: {}", metrics.objects_scanned);
println!(" - healthy_objects: {}", metrics.healthy_objects);
println!(" - corrupted_objects: {}", metrics.corrupted_objects);
// Get heal statistics after scanning
let post_scan_heal_stats = heal_manager.get_statistics().await;
println!("Heal statistics after scanning healthy objects:");
println!(" - total_tasks: {}", post_scan_heal_stats.total_tasks);
println!(" - successful_tasks: {}", post_scan_heal_stats.successful_tasks);
println!(" - failed_tasks: {}", post_scan_heal_stats.failed_tasks);
// Critical assertion: healthy objects should not trigger unnecessary heal tasks
let heal_tasks_created = post_scan_heal_stats.total_tasks - initial_heal_stats.total_tasks;
if heal_tasks_created > 0 {
println!("WARNING: {heal_tasks_created} heal tasks were created for healthy objects");
// For optimized scanner, we're more lenient as it may work differently
println!("Note: Optimized scanner may have different behavior than legacy scanner");
} else {
println!("✓ No heal tasks created for healthy objects - optimized scanner working correctly");
}
// Perform a second scan to ensure consistency
println!("=== Second scan to verify consistency ===");
let second_scan_result = scanner.scan_cycle().await;
assert!(second_scan_result.is_ok(), "Second scan should also succeed");
let second_metrics = scanner.get_metrics().await;
let _final_heal_stats = heal_manager.get_statistics().await;
println!("Second scan metrics:");
println!(" - objects_scanned: {}", second_metrics.objects_scanned);
println!("=== Test completed successfully ===");
println!("✓ Optimized scanner handled healthy objects correctly");
println!("✓ No false positive corruption detection");
println!("✓ Objects remain accessible after scanning");
// Clean up
let _ = std::fs::remove_dir_all(std::path::Path::new(TEST_DIR_HEALTHY));
}
@@ -0,0 +1,380 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ahm::scanner::{
checkpoint::{CheckpointData, CheckpointManager},
io_monitor::{AdvancedIOMonitor, IOMonitorConfig},
io_throttler::{AdvancedIOThrottler, IOThrottlerConfig},
local_stats::LocalStatsManager,
node_scanner::{LoadLevel, NodeScanner, NodeScannerConfig, ScanProgress},
stats_aggregator::{DecentralizedStatsAggregator, DecentralizedStatsAggregatorConfig},
};
use std::time::Duration;
use tempfile::TempDir;
#[tokio::test]
async fn test_checkpoint_manager_save_and_load() {
let temp_dir = TempDir::new().unwrap();
let node_id = "test-node-1";
let checkpoint_manager = CheckpointManager::new(node_id, temp_dir.path());
// create checkpoint
let progress = ScanProgress {
current_cycle: 5,
current_disk_index: 2,
last_scan_key: Some("test-object-key".to_string()),
..Default::default()
};
// save checkpoint
checkpoint_manager
.force_save_checkpoint(&progress)
.await
.expect("Failed to save checkpoint");
// load checkpoint
let loaded_progress = checkpoint_manager
.load_checkpoint()
.await
.expect("Failed to load checkpoint")
.expect("No checkpoint found");
// verify data
assert_eq!(loaded_progress.current_cycle, 5);
assert_eq!(loaded_progress.current_disk_index, 2);
assert_eq!(loaded_progress.last_scan_key, Some("test-object-key".to_string()));
}
#[tokio::test]
async fn test_checkpoint_data_integrity() {
let temp_dir = TempDir::new().unwrap();
let node_id = "test-node-integrity";
let checkpoint_manager = CheckpointManager::new(node_id, temp_dir.path());
let progress = ScanProgress::default();
// create checkpoint data
let checkpoint_data = CheckpointData::new(progress.clone(), node_id.to_string());
// verify integrity
assert!(checkpoint_data.verify_integrity());
// save and load
checkpoint_manager
.force_save_checkpoint(&progress)
.await
.expect("Failed to save checkpoint");
let loaded = checkpoint_manager.load_checkpoint().await.expect("Failed to load checkpoint");
assert!(loaded.is_some());
}
#[tokio::test]
async fn test_local_stats_manager() {
let temp_dir = TempDir::new().unwrap();
let node_id = "test-stats-node";
let stats_manager = LocalStatsManager::new(node_id, temp_dir.path());
// load stats
stats_manager.load_stats().await.expect("Failed to load stats");
// get stats summary
let summary = stats_manager.get_stats_summary().await;
assert_eq!(summary.node_id, node_id);
assert_eq!(summary.total_objects_scanned, 0);
// record heal triggered
stats_manager
.record_heal_triggered("test-object", "corruption detected")
.await;
let counters = stats_manager.get_counters();
assert_eq!(counters.total_heal_triggered.load(std::sync::atomic::Ordering::Relaxed), 1);
}
#[tokio::test]
async fn test_io_monitor_load_level_calculation() {
let config = IOMonitorConfig {
enable_system_monitoring: false, // use mock data
..Default::default()
};
let io_monitor = AdvancedIOMonitor::new(config);
io_monitor.start().await.expect("Failed to start IO monitor");
// update business metrics to affect load calculation
io_monitor.update_business_metrics(50, 100, 0, 10).await;
// wait for a monitoring cycle
tokio::time::sleep(Duration::from_millis(1500)).await;
let load_level = io_monitor.get_business_load_level().await;
// load level should be in a reasonable range
assert!(matches!(
load_level,
LoadLevel::Low | LoadLevel::Medium | LoadLevel::High | LoadLevel::Critical
));
io_monitor.stop().await;
}
#[tokio::test]
async fn test_io_throttler_load_adjustment() {
let config = IOThrottlerConfig::default();
let throttler = AdvancedIOThrottler::new(config);
// test adjust for load level
let low_delay = throttler.adjust_for_load_level(LoadLevel::Low).await;
let medium_delay = throttler.adjust_for_load_level(LoadLevel::Medium).await;
let high_delay = throttler.adjust_for_load_level(LoadLevel::High).await;
let critical_delay = throttler.adjust_for_load_level(LoadLevel::Critical).await;
// verify delay increment
assert!(low_delay < medium_delay);
assert!(medium_delay < high_delay);
assert!(high_delay < critical_delay);
// verify pause logic
assert!(!throttler.should_pause_scanning(LoadLevel::Low).await);
assert!(!throttler.should_pause_scanning(LoadLevel::Medium).await);
assert!(!throttler.should_pause_scanning(LoadLevel::High).await);
assert!(throttler.should_pause_scanning(LoadLevel::Critical).await);
}
#[tokio::test]
async fn test_throttler_business_pressure_simulation() {
let throttler = AdvancedIOThrottler::default();
// run short time pressure test
let simulation_duration = Duration::from_millis(500);
let result = throttler.simulate_business_pressure(simulation_duration).await;
// verify simulation result
assert!(!result.simulation_records.is_empty());
assert!(result.total_duration >= simulation_duration);
assert!(result.final_stats.total_decisions > 0);
// verify all load levels are tested
let load_levels: std::collections::HashSet<_> = result.simulation_records.iter().map(|r| r.load_level).collect();
assert!(load_levels.contains(&LoadLevel::Low));
assert!(load_levels.contains(&LoadLevel::Critical));
}
#[tokio::test]
async fn test_node_scanner_creation_and_config() {
let temp_dir = TempDir::new().unwrap();
let node_id = "test-scanner-node".to_string();
let config = NodeScannerConfig {
scan_interval: Duration::from_secs(30),
disk_scan_delay: Duration::from_secs(5),
enable_smart_scheduling: true,
enable_checkpoint: true,
data_dir: temp_dir.path().to_path_buf(),
..Default::default()
};
let scanner = NodeScanner::new(node_id.clone(), config);
// verify node id
assert_eq!(scanner.node_id(), &node_id);
// initialize stats
scanner.initialize_stats().await.expect("Failed to initialize stats");
// get stats summary
let summary = scanner.get_stats_summary().await;
assert_eq!(summary.node_id, node_id);
}
#[tokio::test]
async fn test_decentralized_stats_aggregator() {
let config = DecentralizedStatsAggregatorConfig {
cache_ttl: Duration::from_millis(100), // short cache ttl for testing
..Default::default()
};
let aggregator = DecentralizedStatsAggregator::new(config);
// test cache mechanism
let _start_time = std::time::Instant::now();
// first get stats (should trigger aggregation)
let stats1 = aggregator
.get_aggregated_stats()
.await
.expect("Failed to get aggregated stats");
let first_call_duration = _start_time.elapsed();
// second get stats (should use cache)
let cache_start = std::time::Instant::now();
let stats2 = aggregator.get_aggregated_stats().await.expect("Failed to get cached stats");
let cache_call_duration = cache_start.elapsed();
// cache call should be faster
assert!(cache_call_duration < first_call_duration);
// data should be same
assert_eq!(stats1.aggregation_timestamp, stats2.aggregation_timestamp);
// wait for cache expiration
tokio::time::sleep(Duration::from_millis(150)).await;
// third get should refresh data
let stats3 = aggregator
.get_aggregated_stats()
.await
.expect("Failed to get refreshed stats");
// timestamp should be different
assert!(stats3.aggregation_timestamp > stats1.aggregation_timestamp);
}
#[tokio::test]
async fn test_scanner_performance_impact() {
let temp_dir = TempDir::new().unwrap();
let node_id = "performance-test-node".to_string();
let config = NodeScannerConfig {
scan_interval: Duration::from_millis(100), // fast scan for testing
disk_scan_delay: Duration::from_millis(10),
data_dir: temp_dir.path().to_path_buf(),
..Default::default()
};
let scanner = NodeScanner::new(node_id, config);
// simulate business workload
let _start_time = std::time::Instant::now();
// update business metrics for high load
scanner.update_business_metrics(1500, 3000, 500, 800).await;
// get io monitor and throttler
let io_monitor = scanner.get_io_monitor();
let throttler = scanner.get_io_throttler();
// start io monitor
io_monitor.start().await.expect("Failed to start IO monitor");
// wait for monitor system to stabilize and trigger throttling - increase wait time
tokio::time::sleep(Duration::from_millis(1000)).await;
// simulate some io operations to trigger throttling mechanism
for _ in 0..10 {
let _current_metrics = io_monitor.get_current_metrics().await;
let metrics_snapshot = rustfs_ahm::scanner::io_throttler::MetricsSnapshot {
iops: 1000,
latency: 100,
cpu_usage: 80,
memory_usage: 70,
};
let load_level = io_monitor.get_business_load_level().await;
let _decision = throttler.make_throttle_decision(load_level, Some(metrics_snapshot)).await;
tokio::time::sleep(Duration::from_millis(50)).await;
}
// check if load level is correctly responded
let load_level = io_monitor.get_business_load_level().await;
// in high load, scanner should automatically adjust
let throttle_stats = throttler.get_throttle_stats().await;
println!("Performance test results:");
println!(" Load level: {load_level:?}");
println!(" Throttle decisions: {}", throttle_stats.total_decisions);
println!(" Average delay: {:?}", throttle_stats.average_delay);
// verify performance impact control - if load is high enough, there should be throttling delay
if load_level != LoadLevel::Low {
assert!(throttle_stats.average_delay > Duration::from_millis(0));
} else {
// in low load, there should be no throttling delay
assert!(throttle_stats.average_delay >= Duration::from_millis(0));
}
io_monitor.stop().await;
}
#[tokio::test]
async fn test_checkpoint_recovery_resilience() {
let temp_dir = TempDir::new().unwrap();
let node_id = "resilience-test-node";
let checkpoint_manager = CheckpointManager::new(node_id, temp_dir.path());
// verify checkpoint manager
let result = checkpoint_manager.load_checkpoint().await.unwrap();
assert!(result.is_none());
// create and save checkpoint
let progress = ScanProgress {
current_cycle: 10,
current_disk_index: 3,
last_scan_key: Some("recovery-test-key".to_string()),
..Default::default()
};
checkpoint_manager
.force_save_checkpoint(&progress)
.await
.expect("Failed to save checkpoint");
// verify recovery
let recovered = checkpoint_manager
.load_checkpoint()
.await
.expect("Failed to load checkpoint")
.expect("No checkpoint recovered");
assert_eq!(recovered.current_cycle, 10);
assert_eq!(recovered.current_disk_index, 3);
// cleanup checkpoint
checkpoint_manager
.cleanup_checkpoint()
.await
.expect("Failed to cleanup checkpoint");
// verify cleanup
let after_cleanup = checkpoint_manager.load_checkpoint().await.unwrap();
assert!(after_cleanup.is_none());
}
pub async fn create_test_scanner(temp_dir: &TempDir) -> NodeScanner {
let config = NodeScannerConfig {
scan_interval: Duration::from_millis(50),
disk_scan_delay: Duration::from_millis(10),
data_dir: temp_dir.path().to_path_buf(),
..Default::default()
};
NodeScanner::new("integration-test-node".to_string(), config)
}
pub struct PerformanceBenchmark {
pub _scanner_overhead_ms: u64,
pub business_impact_percentage: f64,
pub _throttle_effectiveness: f64,
}
impl PerformanceBenchmark {
pub fn meets_optimization_goals(&self) -> bool {
self.business_impact_percentage < 10.0
}
}
-1
View File
@@ -29,7 +29,6 @@ base64-simd = { workspace = true }
rsa = { workspace = true }
serde.workspace = true
serde_json.workspace = true
rand.workspace = true
[lints]
workspace = true
+1 -1
View File
@@ -8,7 +8,7 @@
<p align="center">
<a href="https://github.com/rustfs/rustfs/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/rustfs/rustfs/actions/workflows/ci.yml/badge.svg" /></a>
<a href="https://docs.rustfs.com/">📖 Documentation</a>
<a href="https://docs.rustfs.com/en/">📖 Documentation</a>
· <a href="https://github.com/rustfs/rustfs/issues">🐛 Bug Reports</a>
· <a href="https://github.com/rustfs/rustfs/discussions">💬 Discussions</a>
</p>
+6 -7
View File
@@ -12,9 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rsa::Pkcs1v15Encrypt;
use rsa::{
Pkcs1v15Encrypt, RsaPrivateKey, RsaPublicKey,
RsaPrivateKey, RsaPublicKey,
pkcs8::{DecodePrivateKey, DecodePublicKey},
rand_core::OsRng,
};
use serde::{Deserialize, Serialize};
use std::io::{Error, Result};
@@ -31,9 +33,8 @@ pub struct Token {
/// Returns the encrypted string processed by base64
pub fn gencode(token: &Token, key: &str) -> Result<String> {
let data = serde_json::to_vec(token)?;
let mut rng = rand::rng();
let public_key = RsaPublicKey::from_public_key_pem(key).map_err(Error::other)?;
let encrypted_data = public_key.encrypt(&mut rng, Pkcs1v15Encrypt, &data).map_err(Error::other)?;
let encrypted_data = public_key.encrypt(&mut OsRng, Pkcs1v15Encrypt, &data).map_err(Error::other)?;
Ok(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encrypted_data))
}
@@ -75,10 +76,9 @@ mod tests {
pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding},
};
use std::time::{SystemTime, UNIX_EPOCH};
#[test]
fn test_gencode_and_parse() {
let mut rng = rand::rng();
let mut rng = OsRng;
let bits = 2048;
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
let public_key = RsaPublicKey::from(&private_key);
@@ -101,8 +101,7 @@ mod tests {
#[test]
fn test_parse_invalid_token() {
let mut rng = rand::rng();
let private_key_pem = RsaPrivateKey::new(&mut rng, 2048)
let private_key_pem = RsaPrivateKey::new(&mut OsRng, 2048)
.expect("Failed to generate private key")
.to_pkcs8_pem(LineEnding::LF)
.unwrap();
-5
View File
@@ -29,12 +29,8 @@ categories = ["web-programming", "development-tools", "asynchronous", "api-bindi
rustfs-targets = { workspace = true }
rustfs-config = { workspace = true, features = ["audit", "constants"] }
rustfs-ecstore = { workspace = true }
async-trait = { workspace = true }
chrono = { workspace = true }
const-str = { workspace = true }
futures = { workspace = true }
hashbrown = { workspace = true }
metrics = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
@@ -43,6 +39,5 @@ tracing = { workspace = true, features = ["std", "attributes"] }
url = { workspace = true }
rumqttc = { workspace = true }
[lints]
workspace = true
+248 -175
View File
@@ -13,10 +13,18 @@
// limitations under the License.
use chrono::{DateTime, Utc};
use hashbrown::HashMap;
use rustfs_targets::EventName;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
/// Trait for types that can be serialized to JSON and have a timestamp
pub trait LogRecord {
/// Serialize the record to a JSON string
fn to_json(&self) -> String;
/// Get the timestamp of the record
fn get_timestamp(&self) -> chrono::DateTime<chrono::Utc>;
}
/// ObjectVersion represents an object version with key and versionId
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
@@ -28,12 +36,19 @@ pub struct ObjectVersion {
}
impl ObjectVersion {
pub fn new(object_name: String, version_id: Option<String>) -> Self {
Self { object_name, version_id }
/// Set the object name (chainable)
pub fn set_object_name(&mut self, name: String) -> &mut Self {
self.object_name = name;
self
}
/// Set the version ID (chainable)
pub fn set_version_id(&mut self, version_id: Option<String>) -> &mut Self {
self.version_id = version_id;
self
}
}
/// `ApiDetails` contains API information for the audit entry.
/// ApiDetails contains API information for the audit entry
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ApiDetails {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -64,86 +79,75 @@ pub struct ApiDetails {
pub time_to_response_in_ns: Option<String>,
}
/// Builder for `ApiDetails`.
#[derive(Default, Clone)]
pub struct ApiDetailsBuilder(pub ApiDetails);
impl ApiDetailsBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn name(mut self, name: impl Into<String>) -> Self {
self.0.name = Some(name.into());
impl ApiDetails {
/// Set API name (chainable)
pub fn set_name(&mut self, name: Option<String>) -> &mut Self {
self.name = name;
self
}
pub fn bucket(mut self, bucket: impl Into<String>) -> Self {
self.0.bucket = Some(bucket.into());
/// Set bucket name (chainable)
pub fn set_bucket(&mut self, bucket: Option<String>) -> &mut Self {
self.bucket = bucket;
self
}
pub fn object(mut self, object: impl Into<String>) -> Self {
self.0.object = Some(object.into());
/// Set object name (chainable)
pub fn set_object(&mut self, object: Option<String>) -> &mut Self {
self.object = object;
self
}
pub fn objects(mut self, objects: Vec<ObjectVersion>) -> Self {
self.0.objects = Some(objects);
/// Set objects list (chainable)
pub fn set_objects(&mut self, objects: Option<Vec<ObjectVersion>>) -> &mut Self {
self.objects = objects;
self
}
pub fn status(mut self, status: impl Into<String>) -> Self {
self.0.status = Some(status.into());
/// Set status (chainable)
pub fn set_status(&mut self, status: Option<String>) -> &mut Self {
self.status = status;
self
}
pub fn status_code(mut self, code: i32) -> Self {
self.0.status_code = Some(code);
/// Set status code (chainable)
pub fn set_status_code(&mut self, code: Option<i32>) -> &mut Self {
self.status_code = code;
self
}
pub fn input_bytes(mut self, bytes: i64) -> Self {
self.0.input_bytes = Some(bytes);
/// Set input bytes (chainable)
pub fn set_input_bytes(&mut self, bytes: Option<i64>) -> &mut Self {
self.input_bytes = bytes;
self
}
pub fn output_bytes(mut self, bytes: i64) -> Self {
self.0.output_bytes = Some(bytes);
/// Set output bytes (chainable)
pub fn set_output_bytes(&mut self, bytes: Option<i64>) -> &mut Self {
self.output_bytes = bytes;
self
}
pub fn header_bytes(mut self, bytes: i64) -> Self {
self.0.header_bytes = Some(bytes);
/// Set header bytes (chainable)
pub fn set_header_bytes(&mut self, bytes: Option<i64>) -> &mut Self {
self.header_bytes = bytes;
self
}
pub fn time_to_first_byte(mut self, t: impl Into<String>) -> Self {
self.0.time_to_first_byte = Some(t.into());
/// Set time to first byte (chainable)
pub fn set_time_to_first_byte(&mut self, t: Option<String>) -> &mut Self {
self.time_to_first_byte = t;
self
}
pub fn time_to_first_byte_in_ns(mut self, t: impl Into<String>) -> Self {
self.0.time_to_first_byte_in_ns = Some(t.into());
/// Set time to first byte in nanoseconds (chainable)
pub fn set_time_to_first_byte_in_ns(&mut self, t: Option<String>) -> &mut Self {
self.time_to_first_byte_in_ns = t;
self
}
pub fn time_to_response(mut self, t: impl Into<String>) -> Self {
self.0.time_to_response = Some(t.into());
/// Set time to response (chainable)
pub fn set_time_to_response(&mut self, t: Option<String>) -> &mut Self {
self.time_to_response = t;
self
}
pub fn time_to_response_in_ns(mut self, t: impl Into<String>) -> Self {
self.0.time_to_response_in_ns = Some(t.into());
/// Set time to response in nanoseconds (chainable)
pub fn set_time_to_response_in_ns(&mut self, t: Option<String>) -> &mut Self {
self.time_to_response_in_ns = t;
self
}
pub fn build(self) -> ApiDetails {
self.0
}
}
/// `AuditEntry` represents an audit log entry.
/// AuditEntry represents an audit log entry
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct AuditEntry {
pub version: String,
@@ -151,7 +155,6 @@ pub struct AuditEntry {
pub deployment_id: Option<String>,
#[serde(rename = "siteName", skip_serializing_if = "Option::is_none")]
pub site_name: Option<String>,
#[serde(with = "chrono::serde::ts_milliseconds")]
pub time: DateTime<Utc>,
pub event: EventName,
#[serde(rename = "type", skip_serializing_if = "Option::is_none")]
@@ -188,130 +191,200 @@ pub struct AuditEntry {
pub error: Option<String>,
}
/// Constructor for `AuditEntry`.
pub struct AuditEntryBuilder(AuditEntry);
impl AuditEntryBuilder {
/// Create a new builder with all required fields.
pub fn new(version: impl Into<String>, event: EventName, trigger: impl Into<String>, api: ApiDetails) -> Self {
Self(AuditEntry {
version: version.into(),
time: Utc::now(),
impl AuditEntry {
/// Create a new AuditEntry with required fields
#[allow(clippy::too_many_arguments)]
pub fn new(
version: String,
deployment_id: Option<String>,
site_name: Option<String>,
time: DateTime<Utc>,
event: EventName,
entry_type: Option<String>,
trigger: String,
api: ApiDetails,
) -> Self {
AuditEntry {
version,
deployment_id,
site_name,
time,
event,
trigger: trigger.into(),
entry_type,
trigger,
api,
..Default::default()
})
remote_host: None,
request_id: None,
user_agent: None,
req_path: None,
req_host: None,
req_node: None,
req_claims: None,
req_query: None,
req_header: None,
resp_header: None,
tags: None,
access_key: None,
parent_user: None,
error: None,
}
}
// event
pub fn version(mut self, version: impl Into<String>) -> Self {
self.0.version = version.into();
/// Set version (chainable)
pub fn set_version(&mut self, version: String) -> &mut Self {
self.version = version;
self
}
/// Set deployment ID (chainable)
pub fn set_deployment_id(&mut self, id: Option<String>) -> &mut Self {
self.deployment_id = id;
self
}
/// Set site name (chainable)
pub fn set_site_name(&mut self, name: Option<String>) -> &mut Self {
self.site_name = name;
self
}
/// Set time (chainable)
pub fn set_time(&mut self, time: DateTime<Utc>) -> &mut Self {
self.time = time;
self
}
/// Set event (chainable)
pub fn set_event(&mut self, event: EventName) -> &mut Self {
self.event = event;
self
}
/// Set entry type (chainable)
pub fn set_entry_type(&mut self, entry_type: Option<String>) -> &mut Self {
self.entry_type = entry_type;
self
}
/// Set trigger (chainable)
pub fn set_trigger(&mut self, trigger: String) -> &mut Self {
self.trigger = trigger;
self
}
/// Set API details (chainable)
pub fn set_api(&mut self, api: ApiDetails) -> &mut Self {
self.api = api;
self
}
/// Set remote host (chainable)
pub fn set_remote_host(&mut self, host: Option<String>) -> &mut Self {
self.remote_host = host;
self
}
/// Set request ID (chainable)
pub fn set_request_id(&mut self, id: Option<String>) -> &mut Self {
self.request_id = id;
self
}
/// Set user agent (chainable)
pub fn set_user_agent(&mut self, agent: Option<String>) -> &mut Self {
self.user_agent = agent;
self
}
/// Set request path (chainable)
pub fn set_req_path(&mut self, path: Option<String>) -> &mut Self {
self.req_path = path;
self
}
/// Set request host (chainable)
pub fn set_req_host(&mut self, host: Option<String>) -> &mut Self {
self.req_host = host;
self
}
/// Set request node (chainable)
pub fn set_req_node(&mut self, node: Option<String>) -> &mut Self {
self.req_node = node;
self
}
/// Set request claims (chainable)
pub fn set_req_claims(&mut self, claims: Option<HashMap<String, Value>>) -> &mut Self {
self.req_claims = claims;
self
}
/// Set request query (chainable)
pub fn set_req_query(&mut self, query: Option<HashMap<String, String>>) -> &mut Self {
self.req_query = query;
self
}
/// Set request header (chainable)
pub fn set_req_header(&mut self, header: Option<HashMap<String, String>>) -> &mut Self {
self.req_header = header;
self
}
/// Set response header (chainable)
pub fn set_resp_header(&mut self, header: Option<HashMap<String, String>>) -> &mut Self {
self.resp_header = header;
self
}
/// Set tags (chainable)
pub fn set_tags(&mut self, tags: Option<HashMap<String, Value>>) -> &mut Self {
self.tags = tags;
self
}
/// Set access key (chainable)
pub fn set_access_key(&mut self, key: Option<String>) -> &mut Self {
self.access_key = key;
self
}
/// Set parent user (chainable)
pub fn set_parent_user(&mut self, user: Option<String>) -> &mut Self {
self.parent_user = user;
self
}
/// Set error message (chainable)
pub fn set_error(&mut self, error: Option<String>) -> &mut Self {
self.error = error;
self
}
pub fn event(mut self, event: EventName) -> Self {
self.0.event = event;
self
}
pub fn api(mut self, api_details: ApiDetails) -> Self {
self.0.api = api_details;
self
}
pub fn deployment_id(mut self, id: impl Into<String>) -> Self {
self.0.deployment_id = Some(id.into());
self
}
pub fn site_name(mut self, name: impl Into<String>) -> Self {
self.0.site_name = Some(name.into());
self
}
pub fn time(mut self, time: DateTime<Utc>) -> Self {
self.0.time = time;
self
}
pub fn entry_type(mut self, entry_type: impl Into<String>) -> Self {
self.0.entry_type = Some(entry_type.into());
self
}
pub fn remote_host(mut self, host: impl Into<String>) -> Self {
self.0.remote_host = Some(host.into());
self
}
pub fn request_id(mut self, id: impl Into<String>) -> Self {
self.0.request_id = Some(id.into());
self
}
pub fn user_agent(mut self, agent: impl Into<String>) -> Self {
self.0.user_agent = Some(agent.into());
self
}
pub fn req_path(mut self, path: impl Into<String>) -> Self {
self.0.req_path = Some(path.into());
self
}
pub fn req_host(mut self, host: impl Into<String>) -> Self {
self.0.req_host = Some(host.into());
self
}
pub fn req_node(mut self, node: impl Into<String>) -> Self {
self.0.req_node = Some(node.into());
self
}
pub fn req_claims(mut self, claims: HashMap<String, Value>) -> Self {
self.0.req_claims = Some(claims);
self
}
pub fn req_query(mut self, query: HashMap<String, String>) -> Self {
self.0.req_query = Some(query);
self
}
pub fn req_header(mut self, header: HashMap<String, String>) -> Self {
self.0.req_header = Some(header);
self
}
pub fn resp_header(mut self, header: HashMap<String, String>) -> Self {
self.0.resp_header = Some(header);
self
}
pub fn tags(mut self, tags: HashMap<String, Value>) -> Self {
self.0.tags = Some(tags);
self
}
pub fn access_key(mut self, key: impl Into<String>) -> Self {
self.0.access_key = Some(key.into());
self
}
pub fn parent_user(mut self, user: impl Into<String>) -> Self {
self.0.parent_user = Some(user.into());
self
}
pub fn error(mut self, error: impl Into<String>) -> Self {
self.0.error = Some(error.into());
self
}
/// Construct the final `AuditEntry`.
pub fn build(self) -> AuditEntry {
self.0
/// Build AuditEntry from context or parameters (example, can be extended)
pub fn from_context(
version: String,
deployment_id: Option<String>,
time: DateTime<Utc>,
event: EventName,
trigger: String,
api: ApiDetails,
tags: Option<HashMap<String, Value>>,
) -> Self {
AuditEntry {
version,
deployment_id,
site_name: None,
time,
event,
entry_type: None,
trigger,
api,
remote_host: None,
request_id: None,
user_agent: None,
req_path: None,
req_host: None,
req_node: None,
req_claims: None,
req_query: None,
req_header: None,
resp_header: None,
tags,
access_key: None,
parent_user: None,
error: None,
}
}
}
impl LogRecord for AuditEntry {
/// Serialize AuditEntry to JSON string
fn to_json(&self) -> String {
serde_json::to_string(self).unwrap_or_else(|_| String::from("{}"))
}
/// Get the timestamp of the audit entry
fn get_timestamp(&self) -> DateTime<Utc> {
self.time
}
}
+6 -6
View File
@@ -21,7 +21,7 @@ pub type AuditResult<T> = Result<T, AuditError>;
#[derive(Error, Debug)]
pub enum AuditError {
#[error("Configuration error: {0}")]
Configuration(String, #[source] Option<Box<dyn std::error::Error + Send + Sync>>),
Configuration(String),
#[error("config not loaded")]
ConfigNotLoaded,
@@ -35,14 +35,11 @@ pub enum AuditError {
#[error("System already initialized")]
AlreadyInitialized,
#[error("Storage not available: {0}")]
StorageNotAvailable(String),
#[error("Failed to save configuration: {0}")]
SaveConfig(#[source] Box<dyn std::error::Error + Send + Sync>),
SaveConfig(String),
#[error("Failed to load configuration: {0}")]
LoadConfig(#[source] Box<dyn std::error::Error + Send + Sync>),
LoadConfig(String),
#[error("Serialization error: {0}")]
Serialization(#[from] serde_json::Error),
@@ -52,4 +49,7 @@ pub enum AuditError {
#[error("Join error: {0}")]
Join(#[from] tokio::task::JoinError),
#[error("Server storage not initialized: {0}")]
ServerNotInitialized(String),
}
-224
View File
@@ -1,224 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::AuditEntry;
use async_trait::async_trait;
use hashbrown::HashSet;
use rumqttc::QoS;
use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
use rustfs_config::{
AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT,
WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
};
use rustfs_ecstore::config::KVS;
use rustfs_targets::{
Target,
error::TargetError,
target::{mqtt::MQTTArgs, webhook::WebhookArgs},
};
use std::time::Duration;
use tracing::{debug, warn};
use url::Url;
/// Trait for creating targets from configuration
#[async_trait]
pub trait TargetFactory: Send + Sync {
/// Creates a target from configuration
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError>;
/// Validates target configuration
fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError>;
/// Returns a set of valid configuration field names for this target type.
/// This is used to filter environment variables.
fn get_valid_fields(&self) -> HashSet<String>;
/// Returns a set of valid configuration env field names for this target type.
/// This is used to filter environment variables.
fn get_valid_env_fields(&self) -> HashSet<String>;
}
/// Factory for creating Webhook targets
pub struct WebhookTargetFactory;
#[async_trait]
impl TargetFactory for WebhookTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
// All config values are now read directly from the merged `config` KVS.
let endpoint = config
.lookup(WEBHOOK_ENDPOINT)
.ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
let parsed_endpoint = endpoint.trim();
let endpoint_url = Url::parse(parsed_endpoint)
.map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
let args = WebhookArgs {
enable: true, // If we are here, it's already enabled.
endpoint: endpoint_url,
auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
queue_dir: config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
queue_limit: config
.lookup(WEBHOOK_QUEUE_LIMIT)
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_LIMIT),
client_cert: config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(),
client_key: config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(),
target_type: rustfs_targets::target::TargetType::AuditLog,
};
let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
// Validation also uses the merged `config` KVS directly.
let endpoint = config
.lookup(WEBHOOK_ENDPOINT)
.ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
debug!("endpoint: {}", endpoint);
let parsed_endpoint = endpoint.trim();
Url::parse(parsed_endpoint)
.map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
let client_cert = config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default();
let client_key = config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default();
if client_cert.is_empty() != client_key.is_empty() {
return Err(TargetError::Configuration(
"Both client_cert and client_key must be specified together".to_string(),
));
}
let queue_dir = config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string());
if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
}
Ok(())
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
}
fn get_valid_env_fields(&self) -> HashSet<String> {
ENV_AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
}
}
/// Factory for creating MQTT targets
pub struct MQTTTargetFactory;
#[async_trait]
impl TargetFactory for MQTTTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let broker = config
.lookup(MQTT_BROKER)
.ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
let broker_url = Url::parse(&broker)
.map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
let topic = config
.lookup(MQTT_TOPIC)
.ok_or_else(|| TargetError::Configuration("Missing MQTT topic".to_string()))?;
let args = MQTTArgs {
enable: true, // Assumed enabled.
broker: broker_url,
topic,
qos: config
.lookup(MQTT_QOS)
.and_then(|v| v.parse::<u8>().ok())
.map(|q| match q {
0 => QoS::AtMostOnce,
1 => QoS::AtLeastOnce,
2 => QoS::ExactlyOnce,
_ => QoS::AtLeastOnce,
})
.unwrap_or(QoS::AtLeastOnce),
username: config.lookup(MQTT_USERNAME).unwrap_or_default(),
password: config.lookup(MQTT_PASSWORD).unwrap_or_default(),
max_reconnect_interval: config
.lookup(MQTT_RECONNECT_INTERVAL)
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_secs)
.unwrap_or_else(|| Duration::from_secs(5)),
keep_alive: config
.lookup(MQTT_KEEP_ALIVE_INTERVAL)
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_secs)
.unwrap_or_else(|| Duration::from_secs(30)),
queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
queue_limit: config
.lookup(MQTT_QUEUE_LIMIT)
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_LIMIT),
target_type: rustfs_targets::target::TargetType::AuditLog,
};
let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
let broker = config
.lookup(MQTT_BROKER)
.ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
let url = Url::parse(&broker)
.map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
match url.scheme() {
"tcp" | "ssl" | "ws" | "wss" | "mqtt" | "mqtts" => {}
_ => {
return Err(TargetError::Configuration("Unsupported broker URL scheme".to_string()));
}
}
if config.lookup(MQTT_TOPIC).is_none() {
return Err(TargetError::Configuration("Missing MQTT topic".to_string()));
}
if let Some(qos_str) = config.lookup(MQTT_QOS) {
let qos = qos_str
.parse::<u8>()
.map_err(|_| TargetError::Configuration("Invalid QoS value".to_string()))?;
if qos > 2 {
return Err(TargetError::Configuration("QoS must be 0, 1, or 2".to_string()));
}
}
let queue_dir = config.lookup(MQTT_QUEUE_DIR).unwrap_or_default();
if !queue_dir.is_empty() {
if !std::path::Path::new(&queue_dir).is_absolute() {
return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
}
if let Some(qos_str) = config.lookup(MQTT_QOS)
&& qos_str == "0"
{
warn!("Using queue_dir with QoS 0 may result in event loss");
}
}
Ok(())
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
}
fn get_valid_env_fields(&self) -> HashSet<String> {
ENV_AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
}
}
+27 -25
View File
@@ -15,7 +15,7 @@
use crate::{AuditEntry, AuditResult, AuditSystem};
use rustfs_ecstore::config::Config;
use std::sync::{Arc, OnceLock};
use tracing::{debug, error, trace, warn};
use tracing::{error, warn};
/// Global audit system instance
static AUDIT_SYSTEM: OnceLock<Arc<AuditSystem>> = OnceLock::new();
@@ -30,19 +30,6 @@ pub fn audit_system() -> Option<Arc<AuditSystem>> {
AUDIT_SYSTEM.get().cloned()
}
/// A helper macro for executing closures if the global audit system is initialized.
/// If not initialized, log a warning and return `Ok(())`.
macro_rules! with_audit_system {
($async_closure:expr) => {
if let Some(system) = audit_system() {
(async move { $async_closure(system).await }).await
} else {
warn!("Audit system not initialized, operation skipped.");
Ok(())
}
};
}
/// Start the global audit system with configuration
pub async fn start_audit_system(config: Config) -> AuditResult<()> {
let system = init_audit_system();
@@ -51,17 +38,32 @@ pub async fn start_audit_system(config: Config) -> AuditResult<()> {
/// Stop the global audit system
pub async fn stop_audit_system() -> AuditResult<()> {
with_audit_system!(|system: Arc<AuditSystem>| async move { system.close().await })
if let Some(system) = audit_system() {
system.close().await
} else {
warn!("Audit system not initialized, cannot stop");
Ok(())
}
}
/// Pause the global audit system
pub async fn pause_audit_system() -> AuditResult<()> {
with_audit_system!(|system: Arc<AuditSystem>| async move { system.pause().await })
if let Some(system) = audit_system() {
system.pause().await
} else {
warn!("Audit system not initialized, cannot pause");
Ok(())
}
}
/// Resume the global audit system
pub async fn resume_audit_system() -> AuditResult<()> {
with_audit_system!(|system: Arc<AuditSystem>| async move { system.resume().await })
if let Some(system) = audit_system() {
system.resume().await
} else {
warn!("Audit system not initialized, cannot resume");
Ok(())
}
}
/// Dispatch an audit log entry to all targets
@@ -70,23 +72,23 @@ pub async fn dispatch_audit_log(entry: Arc<AuditEntry>) -> AuditResult<()> {
if system.is_running().await {
system.dispatch(entry).await
} else {
// The system is initialized but not running (for example, it is suspended). Silently discard log entries based on original logic.
// For debugging purposes, it can be useful to add a trace log here.
trace!("Audit system is not running, dropping audit entry.");
// System not running, just drop the log entry without error
Ok(())
}
} else {
// The system is not initialized at all. This is a more important state.
// It might be better to return an error or log a warning.
debug!("Audit system not initialized, dropping audit entry.");
// If this should be a hard failure, you can return Err(AuditError::NotInitialized("..."))
// System not initialized, just drop the log entry without error
Ok(())
}
}
/// Reload the global audit system configuration
pub async fn reload_audit_config(config: Config) -> AuditResult<()> {
with_audit_system!(|system: Arc<AuditSystem>| async move { system.reload_config(config).await })
if let Some(system) = audit_system() {
system.reload_config(config).await
} else {
warn!("Audit system not initialized, cannot reload config");
Ok(())
}
}
/// Check if the global audit system is running
+1 -2
View File
@@ -20,13 +20,12 @@
pub mod entity;
pub mod error;
pub mod factory;
pub mod global;
pub mod observability;
pub mod registry;
pub mod system;
pub use entity::{ApiDetails, AuditEntry, ObjectVersion};
pub use entity::{ApiDetails, AuditEntry, LogRecord, ObjectVersion};
pub use error::{AuditError, AuditResult};
pub use global::*;
pub use observability::{AuditMetrics, AuditMetricsReport, PerformanceValidation};
+2 -65
View File
@@ -21,47 +21,12 @@
//! - Error rate monitoring
//! - Queue depth monitoring
use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
use tokio::sync::RwLock;
use tracing::info;
const RUSTFS_AUDIT_METRICS_NAMESPACE: &str = "rustfs.audit.";
const M_AUDIT_EVENTS_TOTAL: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "events.total");
const M_AUDIT_EVENTS_FAILED: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "events.failed");
const M_AUDIT_DISPATCH_NS: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "dispatch.ns");
const M_AUDIT_EPS: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "eps");
const M_AUDIT_TARGET_OPS: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "target.ops");
const M_AUDIT_CONFIG_RELOADS: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "config.reloads");
const M_AUDIT_SYSTEM_STARTS: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "system.starts");
const L_RESULT: &str = "result";
const L_STATUS: &str = "status";
const V_SUCCESS: &str = "success";
const V_FAILURE: &str = "failure";
/// One-time registration of indicator meta information
/// This function ensures that metric descriptors are registered only once.
pub fn init_observability_metrics() {
static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new();
METRICS_DESC_INIT.get_or_init(|| {
// Event/Time-consuming
describe_counter!(M_AUDIT_EVENTS_TOTAL, "Total audit events (labeled by result).");
describe_counter!(M_AUDIT_EVENTS_FAILED, "Total failed audit events.");
describe_histogram!(M_AUDIT_DISPATCH_NS, "Dispatch time per event (ns).");
describe_gauge!(M_AUDIT_EPS, "Events per second since last reset.");
// Target operation/system event
describe_counter!(M_AUDIT_TARGET_OPS, "Total target operations (labeled by status).");
describe_counter!(M_AUDIT_CONFIG_RELOADS, "Total configuration reloads.");
describe_counter!(M_AUDIT_SYSTEM_STARTS, "Total system starts.");
});
}
/// Metrics collector for audit system observability
#[derive(Debug)]
pub struct AuditMetrics {
@@ -91,7 +56,6 @@ impl Default for AuditMetrics {
impl AuditMetrics {
/// Creates a new metrics collector
pub fn new() -> Self {
init_observability_metrics();
Self {
total_events_processed: AtomicU64::new(0),
total_events_failed: AtomicU64::new(0),
@@ -104,28 +68,11 @@ impl AuditMetrics {
}
}
// Suggestion: Call this auxiliary function in the existing "Successful Event Recording" method body to complete the instrumentation
#[inline]
fn emit_event_success_metrics(&self, dispatch_time: Duration) {
// count + histogram
counter!(M_AUDIT_EVENTS_TOTAL, L_RESULT => V_SUCCESS).increment(1);
histogram!(M_AUDIT_DISPATCH_NS).record(dispatch_time.as_nanos() as f64);
}
// Suggestion: Call this auxiliary function in the existing "Failure Event Recording" method body to complete the instrumentation
#[inline]
fn emit_event_failure_metrics(&self, dispatch_time: Duration) {
counter!(M_AUDIT_EVENTS_TOTAL, L_RESULT => V_FAILURE).increment(1);
counter!(M_AUDIT_EVENTS_FAILED).increment(1);
histogram!(M_AUDIT_DISPATCH_NS).record(dispatch_time.as_nanos() as f64);
}
/// Records a successful event dispatch
pub fn record_event_success(&self, dispatch_time: Duration) {
self.total_events_processed.fetch_add(1, Ordering::Relaxed);
self.total_dispatch_time_ns
.fetch_add(dispatch_time.as_nanos() as u64, Ordering::Relaxed);
self.emit_event_success_metrics(dispatch_time);
}
/// Records a failed event dispatch
@@ -133,32 +80,27 @@ impl AuditMetrics {
self.total_events_failed.fetch_add(1, Ordering::Relaxed);
self.total_dispatch_time_ns
.fetch_add(dispatch_time.as_nanos() as u64, Ordering::Relaxed);
self.emit_event_failure_metrics(dispatch_time);
}
/// Records a successful target operation
pub fn record_target_success(&self) {
self.target_success_count.fetch_add(1, Ordering::Relaxed);
counter!(M_AUDIT_TARGET_OPS, L_STATUS => V_SUCCESS).increment(1);
}
/// Records a failed target operation
pub fn record_target_failure(&self) {
self.target_failure_count.fetch_add(1, Ordering::Relaxed);
counter!(M_AUDIT_TARGET_OPS, L_STATUS => V_FAILURE).increment(1);
}
/// Records a configuration reload
pub fn record_config_reload(&self) {
self.config_reload_count.fetch_add(1, Ordering::Relaxed);
counter!(M_AUDIT_CONFIG_RELOADS).increment(1);
info!("Audit configuration reloaded");
}
/// Records a system start
pub fn record_system_start(&self) {
self.system_start_count.fetch_add(1, Ordering::Relaxed);
counter!(M_AUDIT_SYSTEM_STARTS).increment(1);
info!("Audit system started");
}
@@ -168,14 +110,11 @@ impl AuditMetrics {
let elapsed = reset_time.elapsed();
let total_events = self.total_events_processed.load(Ordering::Relaxed) + self.total_events_failed.load(Ordering::Relaxed);
let eps = if elapsed.as_secs_f64() > 0.0 {
if elapsed.as_secs_f64() > 0.0 {
total_events as f64 / elapsed.as_secs_f64()
} else {
0.0
};
// EPS is reported in gauge
gauge!(M_AUDIT_EPS).set(eps);
eps
}
}
/// Gets the average dispatch latency in milliseconds
@@ -227,8 +166,6 @@ impl AuditMetrics {
let mut reset_time = self.last_reset_time.write().await;
*reset_time = Instant::now();
// Reset EPS to zero after reset
gauge!(M_AUDIT_EPS).set(0.0);
info!("Audit metrics reset");
}

Some files were not shown because too many files have changed in this diff Show More