mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-29 17:48:58 +00:00
Compare commits
32 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 10c949af62 | |||
| 4a3325276d | |||
| c5f6c66f72 | |||
| c7c149975b | |||
| d552210b59 | |||
| 581607da6a | |||
| e95107f7d6 | |||
| a693cb52f3 | |||
| 2c7366038e | |||
| 1cc6dfde87 | |||
| 387f4faf78 | |||
| 0f7093c5f9 | |||
| 6a5c0055e7 | |||
| 76288f2501 | |||
| 3497ccfada | |||
| 24e3d3a2ce | |||
| ebad748cdc | |||
| b7e56ed92c | |||
| 4811632751 | |||
| 374a702f04 | |||
| e369e9f481 | |||
| fe2e4a2274 | |||
| b391272e94 | |||
| c55c7a6373 | |||
| 67f1c371a9 | |||
| d987686c14 | |||
| 48a9707110 | |||
| b89450f54d | |||
| e0c99bced4 | |||
| 130f85a575 | |||
| c42fbed3d2 | |||
| fd539f0f0a |
@@ -0,0 +1,58 @@
|
||||
# GitHub Copilot Rules for RustFS Project
|
||||
|
||||
## Core Rules Reference
|
||||
|
||||
This project follows the comprehensive AI coding rules defined in `.rules.md`. Please refer to that file for the complete set of development guidelines, coding standards, and best practices.
|
||||
|
||||
## Copilot-Specific Configuration
|
||||
|
||||
When using GitHub Copilot for this project, ensure you:
|
||||
|
||||
1. **Review the unified rules**: Always check `.rules.md` for the latest project guidelines
|
||||
2. **Follow branch protection**: Never attempt to commit directly to main/master branch
|
||||
3. **Use English**: All code comments, documentation, and variable names must be in English
|
||||
4. **Clean code practices**: Only make modifications you're confident about
|
||||
5. **Test thoroughly**: Ensure all changes pass formatting, linting, and testing requirements
|
||||
|
||||
## Quick Reference
|
||||
|
||||
### Critical Rules
|
||||
- 🚫 **NEVER commit directly to main/master branch**
|
||||
- ✅ **ALWAYS work on feature branches**
|
||||
- 📝 **ALWAYS use English for code and documentation**
|
||||
- 🧹 **ALWAYS clean up temporary files after use**
|
||||
- 🎯 **ONLY make confident, necessary modifications**
|
||||
|
||||
### Pre-commit Checklist
|
||||
```bash
|
||||
# Before committing, always run:
|
||||
cargo fmt --all
|
||||
cargo clippy --all-targets --all-features -- -D warnings
|
||||
cargo check --all-targets
|
||||
cargo test
|
||||
```
|
||||
|
||||
### Branch Workflow
|
||||
```bash
|
||||
git checkout main
|
||||
git pull origin main
|
||||
git checkout -b feat/your-feature-name
|
||||
# Make your changes
|
||||
git add .
|
||||
git commit -m "feat: your feature description"
|
||||
git push origin feat/your-feature-name
|
||||
gh pr create
|
||||
```
|
||||
|
||||
## Important Notes
|
||||
|
||||
- This file serves as an entry point for GitHub Copilot
|
||||
- All detailed rules and guidelines are maintained in `.rules.md`
|
||||
- Updates to coding standards should be made in `.rules.md` to ensure consistency across all AI tools
|
||||
- When in doubt, always refer to `.rules.md` for authoritative guidance
|
||||
|
||||
## See Also
|
||||
|
||||
- [.rules.md](./.rules.md) - Complete AI coding rules and guidelines
|
||||
- [CONTRIBUTING.md](./CONTRIBUTING.md) - Contribution guidelines
|
||||
- [README.md](./README.md) - Project overview and setup instructions
|
||||
@@ -0,0 +1 @@
|
||||
target
|
||||
@@ -19,9 +19,7 @@ Pull Request Template for RustFS
|
||||
|
||||
## Checklist
|
||||
- [ ] I have read and followed the [CONTRIBUTING.md](CONTRIBUTING.md) guidelines
|
||||
- [ ] Code is formatted with `cargo fmt --all`
|
||||
- [ ] Passed `cargo clippy --all-targets --all-features -- -D warnings`
|
||||
- [ ] Passed `cargo check --all-targets`
|
||||
- [ ] Passed `make pre-commit`
|
||||
- [ ] Added/updated necessary tests
|
||||
- [ ] Documentation updated (if needed)
|
||||
- [ ] CI/CD passed (if applicable)
|
||||
|
||||
@@ -16,13 +16,13 @@ name: Security Audit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- '**/Cargo.toml'
|
||||
- '**/Cargo.lock'
|
||||
- '.github/workflows/audit.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- '**/Cargo.toml'
|
||||
- '**/Cargo.lock'
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@v2
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Dependency Review
|
||||
uses: actions/dependency-review-action@v4
|
||||
|
||||
+15
-15
@@ -28,8 +28,8 @@ name: Build and Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ["*.*.*"]
|
||||
branches: [main]
|
||||
tags: [ "*.*.*" ]
|
||||
branches: [ main ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "**.txt"
|
||||
@@ -45,7 +45,7 @@ on:
|
||||
- ".gitignore"
|
||||
- ".dockerignore"
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches: [ main ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "**.txt"
|
||||
@@ -89,7 +89,7 @@ jobs:
|
||||
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -153,7 +153,7 @@ jobs:
|
||||
# Build RustFS binaries
|
||||
build-rustfs:
|
||||
name: Build RustFS
|
||||
needs: [build-check]
|
||||
needs: [ build-check ]
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 60
|
||||
@@ -200,7 +200,7 @@ jobs:
|
||||
# platform: windows
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -527,7 +527,7 @@ jobs:
|
||||
# Build summary
|
||||
build-summary:
|
||||
name: Build Summary
|
||||
needs: [build-check, build-rustfs]
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: always() && needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -579,7 +579,7 @@ jobs:
|
||||
# Create GitHub Release (only for tag pushes)
|
||||
create-release:
|
||||
name: Create GitHub Release
|
||||
needs: [build-check, build-rustfs]
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
@@ -589,7 +589,7 @@ jobs:
|
||||
release_url: ${{ steps.create.outputs.release_url }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -665,7 +665,7 @@ jobs:
|
||||
# Prepare and upload release assets
|
||||
upload-release-assets:
|
||||
name: Upload Release Assets
|
||||
needs: [build-check, build-rustfs, create-release]
|
||||
needs: [ build-check, build-rustfs, create-release ]
|
||||
if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
@@ -673,10 +673,10 @@ jobs:
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
uses: actions/download-artifact@v5
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
@@ -746,7 +746,7 @@ jobs:
|
||||
# Update latest.json for stable releases only
|
||||
update-latest-version:
|
||||
name: Update Latest Version
|
||||
needs: [build-check, upload-release-assets]
|
||||
needs: [ build-check, upload-release-assets ]
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -796,14 +796,14 @@ jobs:
|
||||
# Publish release (remove draft status)
|
||||
publish-release:
|
||||
name: Publish Release
|
||||
needs: [build-check, create-release, upload-release-assets]
|
||||
needs: [ build-check, create-release, upload-release-assets ]
|
||||
if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Update release notes and publish
|
||||
env:
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Continuous Integration
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [ main ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "**.txt"
|
||||
@@ -36,7 +36,7 @@ on:
|
||||
- ".github/workflows/audit.yml"
|
||||
- ".github/workflows/performance.yml"
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches: [ main ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "**.txt"
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
name: Typos
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Typos check with custom config file
|
||||
uses: crate-ci/typos@master
|
||||
@@ -101,7 +101,7 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -130,7 +130,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
|
||||
@@ -36,8 +36,8 @@ permissions:
|
||||
on:
|
||||
# Automatically triggered when build workflow completes
|
||||
workflow_run:
|
||||
workflows: ["Build and Release"]
|
||||
types: [completed]
|
||||
workflows: [ "Build and Release" ]
|
||||
types: [ completed ]
|
||||
# Manual trigger with same parameters for consistency
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
create_latest: ${{ steps.check.outputs.create_latest }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# For workflow_run events, checkout the specific commit that triggered the workflow
|
||||
@@ -250,7 +250,7 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
@@ -382,7 +382,7 @@ jobs:
|
||||
# Docker build summary
|
||||
docker-summary:
|
||||
name: Docker Build Summary
|
||||
needs: [build-check, build-docker]
|
||||
needs: [ build-check, build-docker ]
|
||||
if: always() && needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Performance Testing
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- "**/*.rs"
|
||||
- "**/Cargo.toml"
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -116,7 +116,7 @@ jobs:
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
|
||||
@@ -20,3 +20,4 @@ profile.json
|
||||
.docker/openobserve-otel/data
|
||||
*.zst
|
||||
.secrets
|
||||
*.go
|
||||
@@ -0,0 +1,702 @@
|
||||
# RustFS Project AI Coding Rules
|
||||
|
||||
## 🚨🚨🚨 CRITICAL DEVELOPMENT RULES - ZERO TOLERANCE 🚨🚨🚨
|
||||
|
||||
### ⛔️ ABSOLUTE PROHIBITION: NEVER COMMIT DIRECTLY TO MASTER/MAIN BRANCH ⛔️
|
||||
|
||||
**🔥 THIS IS THE MOST CRITICAL RULE - VIOLATION WILL RESULT IN IMMEDIATE REVERSAL 🔥**
|
||||
|
||||
- **🚫 ZERO DIRECT COMMITS TO MAIN/MASTER BRANCH - ABSOLUTELY FORBIDDEN**
|
||||
- **🚫 ANY DIRECT COMMIT TO MAIN BRANCH MUST BE IMMEDIATELY REVERTED**
|
||||
- **🚫 NO EXCEPTIONS FOR HOTFIXES, EMERGENCIES, OR URGENT CHANGES**
|
||||
- **🚫 NO EXCEPTIONS FOR SMALL CHANGES, TYPOS, OR DOCUMENTATION UPDATES**
|
||||
- **🚫 NO EXCEPTIONS FOR ANYONE - MAINTAINERS, CONTRIBUTORS, OR ADMINS**
|
||||
|
||||
### 📋 MANDATORY WORKFLOW - STRICTLY ENFORCED
|
||||
|
||||
**EVERY SINGLE CHANGE MUST FOLLOW THIS WORKFLOW:**
|
||||
|
||||
1. **Check current branch**: `git branch` (MUST NOT be on main/master)
|
||||
2. **Switch to main**: `git checkout main`
|
||||
3. **Pull latest**: `git pull origin main`
|
||||
4. **Create feature branch**: `git checkout -b feat/your-feature-name`
|
||||
5. **Make changes ONLY on feature branch**
|
||||
6. **Test thoroughly before committing**
|
||||
7. **Commit and push to feature branch**: `git push origin feat/your-feature-name`
|
||||
8. **Create Pull Request**: Use `gh pr create` (MANDATORY)
|
||||
9. **Wait for PR approval**: NO self-merging allowed
|
||||
10. **Merge through GitHub interface**: ONLY after approval
|
||||
|
||||
### 🔒 ENFORCEMENT MECHANISMS
|
||||
|
||||
- **Branch protection rules**: Main branch is protected
|
||||
- **Pre-commit hooks**: Will block direct commits to main
|
||||
- **CI/CD checks**: All PRs must pass before merging
|
||||
- **Code review requirement**: At least one approval needed
|
||||
- **Automated reversal**: Direct commits to main will be automatically reverted
|
||||
|
||||
## 🎯 Core AI Development Principles
|
||||
|
||||
### Five Execution Steps
|
||||
|
||||
#### 1. Task Analysis and Planning
|
||||
- **Clear Objectives**: Deeply understand task requirements and expected results before starting coding
|
||||
- **Plan Development**: List specific files, components, and functions that need modification, explaining the reasons for changes
|
||||
- **Risk Assessment**: Evaluate the impact of changes on existing functionality, develop rollback plans
|
||||
|
||||
#### 2. Precise Code Location
|
||||
- **File Identification**: Determine specific files and line numbers that need modification
|
||||
- **Impact Analysis**: Avoid modifying irrelevant files, clearly state the reason for each file modification
|
||||
- **Minimization Principle**: Unless explicitly required by the task, do not create new abstraction layers or refactor existing code
|
||||
|
||||
#### 3. Minimal Code Changes
|
||||
- **Focus on Core**: Only write code directly required by the task
|
||||
- **Avoid Redundancy**: Do not add unnecessary logs, comments, tests, or error handling
|
||||
- **Isolation**: Ensure new code does not interfere with existing functionality, maintain code independence
|
||||
|
||||
#### 4. Strict Code Review
|
||||
- **Correctness Check**: Verify the correctness and completeness of code logic
|
||||
- **Style Consistency**: Ensure code conforms to established project coding style
|
||||
- **Side Effect Assessment**: Evaluate the impact of changes on downstream systems
|
||||
|
||||
#### 5. Clear Delivery Documentation
|
||||
- **Change Summary**: Detailed explanation of all modifications and reasons
|
||||
- **File List**: List all modified files and their specific changes
|
||||
- **Risk Statement**: Mark any assumptions or potential risk points
|
||||
|
||||
### Core Principles
|
||||
- **🎯 Precise Execution**: Strictly follow task requirements, no arbitrary innovation
|
||||
- **⚡ Efficient Development**: Avoid over-design, only do necessary work
|
||||
- **🛡️ Safe and Reliable**: Always follow development processes, ensure code quality and system stability
|
||||
- **🔒 Cautious Modification**: Only modify when clearly knowing what needs to be changed and having confidence
|
||||
|
||||
### Additional AI Behavior Rules
|
||||
|
||||
1. **Use English for all code comments and documentation** - All comments, variable names, function names, documentation, and user-facing text in code should be in English
|
||||
2. **Clean up temporary scripts after use** - Any temporary scripts, test files, or helper files created during AI work should be removed after task completion
|
||||
3. **Only make confident modifications** - Do not make speculative changes or "convenient" modifications outside the task scope. If uncertain about a change, ask for clarification rather than guessing
|
||||
|
||||
## Project Overview
|
||||
|
||||
RustFS is a high-performance distributed object storage system written in Rust, compatible with S3 API. The project adopts a modular architecture, supporting erasure coding storage, multi-tenant management, observability, and other enterprise-level features.
|
||||
|
||||
## Core Architecture Principles
|
||||
|
||||
### 1. Modular Design
|
||||
|
||||
- Project uses Cargo workspace structure, containing multiple independent crates
|
||||
- Core modules: `rustfs` (main service), `ecstore` (erasure coding storage), `common` (shared components)
|
||||
- Functional modules: `iam` (identity management), `madmin` (management interface), `crypto` (encryption), etc.
|
||||
- Tool modules: `cli` (command line tool), `crates/*` (utility libraries)
|
||||
|
||||
### 2. Asynchronous Programming Pattern
|
||||
|
||||
- Comprehensive use of `tokio` async runtime
|
||||
- Prioritize `async/await` syntax
|
||||
- Use `async-trait` for async methods in traits
|
||||
- Avoid blocking operations, use `spawn_blocking` when necessary
|
||||
|
||||
### 3. Error Handling Strategy
|
||||
|
||||
- **Use modular, type-safe error handling with `thiserror`**
|
||||
- Each module should define its own error type using `thiserror::Error` derive macro
|
||||
- Support error chains and context information through `#[from]` and `#[source]` attributes
|
||||
- Use `Result<T>` type aliases for consistency within each module
|
||||
- Error conversion between modules should use explicit `From` implementations
|
||||
- Follow the pattern: `pub type Result<T> = core::result::Result<T, Error>`
|
||||
- Use `#[error("description")]` attributes for clear error messages
|
||||
- Support error downcasting when needed through `other()` helper methods
|
||||
- Implement `Clone` for errors when required by the domain logic
|
||||
|
||||
## Code Style Guidelines
|
||||
|
||||
### 1. Formatting Configuration
|
||||
|
||||
```toml
|
||||
max_width = 130
|
||||
fn_call_width = 90
|
||||
single_line_let_else_max_width = 100
|
||||
```
|
||||
|
||||
### 2. **🔧 MANDATORY Code Formatting Rules**
|
||||
|
||||
**CRITICAL**: All code must be properly formatted before committing. This project enforces strict formatting standards to maintain code consistency and readability.
|
||||
|
||||
#### Pre-commit Requirements (MANDATORY)
|
||||
|
||||
Before every commit, you **MUST**:
|
||||
|
||||
1. **Format your code**:
|
||||
```bash
|
||||
cargo fmt --all
|
||||
```
|
||||
|
||||
2. **Verify formatting**:
|
||||
```bash
|
||||
cargo fmt --all --check
|
||||
```
|
||||
|
||||
3. **Pass clippy checks**:
|
||||
```bash
|
||||
cargo clippy --all-targets --all-features -- -D warnings
|
||||
```
|
||||
|
||||
4. **Ensure compilation**:
|
||||
```bash
|
||||
cargo check --all-targets
|
||||
```
|
||||
|
||||
#### Quick Commands
|
||||
|
||||
Use these convenient Makefile targets for common tasks:
|
||||
|
||||
```bash
|
||||
# Format all code
|
||||
make fmt
|
||||
|
||||
# Check if code is properly formatted
|
||||
make fmt-check
|
||||
|
||||
# Run clippy checks
|
||||
make clippy
|
||||
|
||||
# Run compilation check
|
||||
make check
|
||||
|
||||
# Run tests
|
||||
make test
|
||||
|
||||
# Run all pre-commit checks (format + clippy + check + test)
|
||||
make pre-commit
|
||||
|
||||
# Setup git hooks (one-time setup)
|
||||
make setup-hooks
|
||||
```
|
||||
|
||||
### 3. Naming Conventions
|
||||
|
||||
- Use `snake_case` for functions, variables, modules
|
||||
- Use `PascalCase` for types, traits, enums
|
||||
- Constants use `SCREAMING_SNAKE_CASE`
|
||||
- Global variables prefix `GLOBAL_`, e.g., `GLOBAL_Endpoints`
|
||||
- Use meaningful and descriptive names for variables, functions, and methods
|
||||
- Avoid meaningless names like `temp`, `data`, `foo`, `bar`, `test123`
|
||||
- Choose names that clearly express the purpose and intent
|
||||
|
||||
### 4. Type Declaration Guidelines
|
||||
|
||||
- **Prefer type inference over explicit type declarations** when the type is obvious from context
|
||||
- Let the Rust compiler infer types whenever possible to reduce verbosity and improve maintainability
|
||||
- Only specify types explicitly when:
|
||||
- The type cannot be inferred by the compiler
|
||||
- Explicit typing improves code clarity and readability
|
||||
- Required for API boundaries (function signatures, public struct fields)
|
||||
- Needed to resolve ambiguity between multiple possible types
|
||||
|
||||
### 5. Documentation Comments
|
||||
|
||||
- Public APIs must have documentation comments
|
||||
- Use `///` for documentation comments
|
||||
- Complex functions add `# Examples` and `# Parameters` descriptions
|
||||
- Error cases use `# Errors` descriptions
|
||||
- Always use English for all comments and documentation
|
||||
- Avoid meaningless comments like "debug 111" or placeholder text
|
||||
|
||||
### 6. Import Guidelines
|
||||
|
||||
- Standard library imports first
|
||||
- Third-party crate imports in the middle
|
||||
- Project internal imports last
|
||||
- Group `use` statements with blank lines between groups
|
||||
|
||||
## Asynchronous Programming Guidelines
|
||||
|
||||
### 1. Trait Definition
|
||||
|
||||
```rust
|
||||
#[async_trait::async_trait]
|
||||
pub trait StorageAPI: Send + Sync {
|
||||
async fn get_object(&self, bucket: &str, object: &str) -> Result<ObjectInfo>;
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Error Handling
|
||||
|
||||
```rust
|
||||
// Use ? operator to propagate errors
|
||||
async fn example_function() -> Result<()> {
|
||||
let data = read_file("path").await?;
|
||||
process_data(data).await?;
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Concurrency Control
|
||||
|
||||
- Use `Arc` and `Mutex`/`RwLock` for shared state management
|
||||
- Prioritize async locks from `tokio::sync`
|
||||
- Avoid holding locks for long periods
|
||||
|
||||
## Logging and Tracing Guidelines
|
||||
|
||||
### 1. Tracing Usage
|
||||
|
||||
```rust
|
||||
#[tracing::instrument(skip(self, data))]
|
||||
async fn process_data(&self, data: &[u8]) -> Result<()> {
|
||||
info!("Processing {} bytes", data.len());
|
||||
// Implementation logic
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Log Levels
|
||||
|
||||
- `error!`: System errors requiring immediate attention
|
||||
- `warn!`: Warning information that may affect functionality
|
||||
- `info!`: Important business information
|
||||
- `debug!`: Debug information for development use
|
||||
- `trace!`: Detailed execution paths
|
||||
|
||||
### 3. Structured Logging
|
||||
|
||||
```rust
|
||||
info!(
|
||||
counter.rustfs_api_requests_total = 1_u64,
|
||||
key_request_method = %request.method(),
|
||||
key_request_uri_path = %request.uri().path(),
|
||||
"API request processed"
|
||||
);
|
||||
```
|
||||
|
||||
## Error Handling Guidelines
|
||||
|
||||
### 1. Error Type Definition
|
||||
|
||||
```rust
|
||||
// Use thiserror for module-specific error types
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
pub enum MyError {
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("Storage error: {0}")]
|
||||
Storage(#[from] ecstore::error::StorageError),
|
||||
|
||||
#[error("Custom error: {message}")]
|
||||
Custom { message: String },
|
||||
|
||||
#[error("File not found: {path}")]
|
||||
FileNotFound { path: String },
|
||||
|
||||
#[error("Invalid configuration: {0}")]
|
||||
InvalidConfig(String),
|
||||
}
|
||||
|
||||
// Provide Result type alias for the module
|
||||
pub type Result<T> = core::result::Result<T, MyError>;
|
||||
```
|
||||
|
||||
### 2. Error Helper Methods
|
||||
|
||||
```rust
|
||||
impl MyError {
|
||||
/// Create error from any compatible error type
|
||||
pub fn other<E>(error: E) -> Self
|
||||
where
|
||||
E: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
MyError::Io(std::io::Error::other(error))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Error Context and Propagation
|
||||
|
||||
```rust
|
||||
// Use ? operator for clean error propagation
|
||||
async fn example_function() -> Result<()> {
|
||||
let data = read_file("path").await?;
|
||||
process_data(data).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Add context to errors
|
||||
fn process_with_context(path: &str) -> Result<()> {
|
||||
std::fs::read(path)
|
||||
.map_err(|e| MyError::Custom {
|
||||
message: format!("Failed to read {}: {}", path, e)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Optimization Guidelines
|
||||
|
||||
### 1. Memory Management
|
||||
|
||||
- Use `Bytes` instead of `Vec<u8>` for zero-copy operations
|
||||
- Avoid unnecessary cloning, use reference passing
|
||||
- Use `Arc` for sharing large objects
|
||||
|
||||
### 2. Concurrency Optimization
|
||||
|
||||
```rust
|
||||
// Use join_all for concurrent operations
|
||||
let futures = disks.iter().map(|disk| disk.operation());
|
||||
let results = join_all(futures).await;
|
||||
```
|
||||
|
||||
### 3. Caching Strategy
|
||||
|
||||
- Use `LazyLock` for global caching
|
||||
- Implement LRU cache to avoid memory leaks
|
||||
|
||||
## Testing Guidelines
|
||||
|
||||
### 1. Unit Tests
|
||||
|
||||
```rust
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use test_case::test_case;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_async_function() {
|
||||
let result = async_function().await;
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test_case("input1", "expected1")]
|
||||
#[test_case("input2", "expected2")]
|
||||
fn test_with_cases(input: &str, expected: &str) {
|
||||
assert_eq!(function(input), expected);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Integration Tests
|
||||
|
||||
- Use `e2e_test` module for end-to-end testing
|
||||
- Simulate real storage environments
|
||||
|
||||
### 3. Test Quality Standards
|
||||
|
||||
- Write meaningful test cases that verify actual functionality
|
||||
- Avoid placeholder or debug content like "debug 111", "test test", etc.
|
||||
- Use descriptive test names that clearly indicate what is being tested
|
||||
- Each test should have a clear purpose and verify specific behavior
|
||||
- Test data should be realistic and representative of actual use cases
|
||||
|
||||
## Cross-Platform Compatibility Guidelines
|
||||
|
||||
### 1. CPU Architecture Compatibility
|
||||
|
||||
- **Always consider multi-platform and different CPU architecture compatibility** when writing code
|
||||
- Support major architectures: x86_64, aarch64 (ARM64), and other target platforms
|
||||
- Use conditional compilation for architecture-specific code:
|
||||
|
||||
```rust
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
fn optimized_x86_64_function() { /* x86_64 specific implementation */ }
|
||||
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
fn optimized_aarch64_function() { /* ARM64 specific implementation */ }
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
|
||||
fn generic_function() { /* Generic fallback implementation */ }
|
||||
```
|
||||
|
||||
### 2. Platform-Specific Dependencies
|
||||
|
||||
- Use feature flags for platform-specific dependencies
|
||||
- Provide fallback implementations for unsupported platforms
|
||||
- Test on multiple architectures in CI/CD pipeline
|
||||
|
||||
### 3. Endianness Considerations
|
||||
|
||||
- Use explicit byte order conversion when dealing with binary data
|
||||
- Prefer `to_le_bytes()`, `from_le_bytes()` for consistent little-endian format
|
||||
- Use `byteorder` crate for complex binary format handling
|
||||
|
||||
### 4. SIMD and Performance Optimizations
|
||||
|
||||
- Use portable SIMD libraries like `wide` or `packed_simd`
|
||||
- Provide fallback implementations for non-SIMD architectures
|
||||
- Use runtime feature detection when appropriate
|
||||
|
||||
## Security Guidelines
|
||||
|
||||
### 1. Memory Safety
|
||||
|
||||
- Disable `unsafe` code (workspace.lints.rust.unsafe_code = "deny")
|
||||
- Use `rustls` instead of `openssl`
|
||||
|
||||
### 2. Authentication and Authorization
|
||||
|
||||
```rust
|
||||
// Use IAM system for permission checks
|
||||
let identity = iam.authenticate(&access_key, &secret_key).await?;
|
||||
iam.authorize(&identity, &action, &resource).await?;
|
||||
```
|
||||
|
||||
## Configuration Management Guidelines
|
||||
|
||||
### 1. Environment Variables
|
||||
|
||||
- Use `RUSTFS_` prefix
|
||||
- Support both configuration files and environment variables
|
||||
- Provide reasonable default values
|
||||
|
||||
### 2. Configuration Structure
|
||||
|
||||
```rust
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct Config {
|
||||
pub address: String,
|
||||
pub volumes: String,
|
||||
#[serde(default)]
|
||||
pub console_enable: bool,
|
||||
}
|
||||
```
|
||||
|
||||
## Dependency Management Guidelines
|
||||
|
||||
### 1. Workspace Dependencies
|
||||
|
||||
- Manage versions uniformly at workspace level
|
||||
- Use `workspace = true` to inherit configuration
|
||||
|
||||
### 2. Feature Flags
|
||||
|
||||
```rust
|
||||
[features]
|
||||
default = ["file"]
|
||||
gpu = ["dep:nvml-wrapper"]
|
||||
kafka = ["dep:rdkafka"]
|
||||
```
|
||||
|
||||
## Deployment and Operations Guidelines
|
||||
|
||||
### 1. Containerization
|
||||
|
||||
- Provide Dockerfile and docker-compose configuration
|
||||
- Support multi-stage builds to optimize image size
|
||||
|
||||
### 2. Observability
|
||||
|
||||
- Integrate OpenTelemetry for distributed tracing
|
||||
- Support Prometheus metrics collection
|
||||
- Provide Grafana dashboards
|
||||
|
||||
### 3. Health Checks
|
||||
|
||||
```rust
|
||||
// Implement health check endpoint
|
||||
async fn health_check() -> Result<HealthStatus> {
|
||||
// Check component status
|
||||
}
|
||||
```
|
||||
|
||||
## Code Review Checklist
|
||||
|
||||
### 1. **Code Formatting and Quality (MANDATORY)**
|
||||
|
||||
- [ ] **Code is properly formatted** (`cargo fmt --all --check` passes)
|
||||
- [ ] **All clippy warnings are resolved** (`cargo clippy --all-targets --all-features -- -D warnings` passes)
|
||||
- [ ] **Code compiles successfully** (`cargo check --all-targets` passes)
|
||||
- [ ] **Pre-commit hooks are working** and all checks pass
|
||||
- [ ] **No formatting-related changes** mixed with functional changes (separate commits)
|
||||
|
||||
### 2. Functionality
|
||||
|
||||
- [ ] Are all error cases properly handled?
|
||||
- [ ] Is there appropriate logging?
|
||||
- [ ] Is there necessary test coverage?
|
||||
|
||||
### 3. Performance
|
||||
|
||||
- [ ] Are unnecessary memory allocations avoided?
|
||||
- [ ] Are async operations used correctly?
|
||||
- [ ] Are there potential deadlock risks?
|
||||
|
||||
### 4. Security
|
||||
|
||||
- [ ] Are input parameters properly validated?
|
||||
- [ ] Are there appropriate permission checks?
|
||||
- [ ] Is information leakage avoided?
|
||||
|
||||
### 5. Cross-Platform Compatibility
|
||||
|
||||
- [ ] Does the code work on different CPU architectures (x86_64, aarch64)?
|
||||
- [ ] Are platform-specific features properly gated with conditional compilation?
|
||||
- [ ] Is byte order handling correct for binary data?
|
||||
- [ ] Are there appropriate fallback implementations for unsupported platforms?
|
||||
|
||||
### 6. Code Commits and Documentation
|
||||
|
||||
- [ ] Does it comply with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)?
|
||||
- [ ] Are commit messages concise and under 72 characters for the title line?
|
||||
- [ ] Commit titles should be concise and in English, avoid Chinese
|
||||
- [ ] Is PR description provided in copyable markdown format for easy copying?
|
||||
|
||||
## Common Patterns and Best Practices
|
||||
|
||||
### 1. Resource Management
|
||||
|
||||
```rust
|
||||
// Use RAII pattern for resource management
|
||||
pub struct ResourceGuard {
|
||||
resource: Resource,
|
||||
}
|
||||
|
||||
impl Drop for ResourceGuard {
|
||||
fn drop(&mut self) {
|
||||
// Clean up resources
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Dependency Injection
|
||||
|
||||
```rust
|
||||
// Use dependency injection pattern
|
||||
pub struct Service {
|
||||
config: Arc<Config>,
|
||||
storage: Arc<dyn StorageAPI>,
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Graceful Shutdown
|
||||
|
||||
```rust
|
||||
// Implement graceful shutdown
|
||||
async fn shutdown_gracefully(shutdown_rx: &mut Receiver<()>) {
|
||||
tokio::select! {
|
||||
_ = shutdown_rx.recv() => {
|
||||
info!("Received shutdown signal");
|
||||
// Perform cleanup operations
|
||||
}
|
||||
_ = tokio::time::sleep(SHUTDOWN_TIMEOUT) => {
|
||||
warn!("Shutdown timeout reached");
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Domain-Specific Guidelines
|
||||
|
||||
### 1. Storage Operations
|
||||
|
||||
- All storage operations must support erasure coding
|
||||
- Implement read/write quorum mechanisms
|
||||
- Support data integrity verification
|
||||
|
||||
### 2. Network Communication
|
||||
|
||||
- Use gRPC for internal service communication
|
||||
- HTTP/HTTPS support for S3-compatible API
|
||||
- Implement connection pooling and retry mechanisms
|
||||
|
||||
### 3. Metadata Management
|
||||
|
||||
- Use FlatBuffers for serialization
|
||||
- Support version control and migration
|
||||
- Implement metadata caching
|
||||
|
||||
## Branch Management and Development Workflow
|
||||
|
||||
### Branch Management
|
||||
|
||||
- **🚨 CRITICAL: NEVER modify code directly on main or master branch - THIS IS ABSOLUTELY FORBIDDEN 🚨**
|
||||
- **⚠️ ANY DIRECT COMMITS TO MASTER/MAIN WILL BE REJECTED AND MUST BE REVERTED IMMEDIATELY ⚠️**
|
||||
- **🔒 ALL CHANGES MUST GO THROUGH PULL REQUESTS - NO DIRECT COMMITS TO MAIN UNDER ANY CIRCUMSTANCES 🔒**
|
||||
- **Always work on feature branches - NO EXCEPTIONS**
|
||||
- Always check the .rules.md file before starting to ensure you understand the project guidelines
|
||||
- **MANDATORY workflow for ALL changes:**
|
||||
1. `git checkout main` (switch to main branch)
|
||||
2. `git pull` (get latest changes)
|
||||
3. `git checkout -b feat/your-feature-name` (create and switch to feature branch)
|
||||
4. Make your changes ONLY on the feature branch
|
||||
5. Test thoroughly before committing
|
||||
6. Commit and push to the feature branch
|
||||
7. **Create a pull request for code review - THIS IS THE ONLY WAY TO MERGE TO MAIN**
|
||||
8. **Wait for PR approval before merging - NEVER merge your own PRs without review**
|
||||
- Use descriptive branch names following the pattern: `feat/feature-name`, `fix/issue-name`, `refactor/component-name`, etc.
|
||||
- **Double-check current branch before ANY commit: `git branch` to ensure you're NOT on main/master**
|
||||
- **Pull Request Requirements:**
|
||||
- All changes must be submitted via PR regardless of size or urgency
|
||||
- PRs must include comprehensive description and testing information
|
||||
- PRs must pass all CI/CD checks before merging
|
||||
- PRs require at least one approval from code reviewers
|
||||
- Even hotfixes and emergency changes must go through PR process
|
||||
- **Enforcement:**
|
||||
- Main branch should be protected with branch protection rules
|
||||
- Direct pushes to main should be blocked by repository settings
|
||||
- Any accidental direct commits to main must be immediately reverted via PR
|
||||
|
||||
### Development Workflow
|
||||
|
||||
## 🎯 **Core Development Principles**
|
||||
|
||||
- **🔴 Every change must be precise - don't modify unless you're confident**
|
||||
- Carefully analyze code logic and ensure complete understanding before making changes
|
||||
- When uncertain, prefer asking users or consulting documentation over blind modifications
|
||||
- Use small iterative steps, modify only necessary parts at a time
|
||||
- Evaluate impact scope before changes to ensure no new issues are introduced
|
||||
|
||||
- **🚀 GitHub PR creation prioritizes gh command usage**
|
||||
- Prefer using `gh pr create` command to create Pull Requests
|
||||
- Avoid having users manually create PRs through web interface
|
||||
- Provide clear and professional PR titles and descriptions
|
||||
- Using `gh` commands ensures better integration and automation
|
||||
|
||||
## 📝 **Code Quality Requirements**
|
||||
|
||||
- Use English for all code comments, documentation, and variable names
|
||||
- Write meaningful and descriptive names for variables, functions, and methods
|
||||
- Avoid meaningless test content like "debug 111" or placeholder values
|
||||
- Before each change, carefully read the existing code to ensure you understand the code structure and implementation, do not break existing logic implementation, do not introduce new issues
|
||||
- Ensure each change provides sufficient test cases to guarantee code correctness
|
||||
- Do not arbitrarily modify numbers and constants in test cases, carefully analyze their meaning to ensure test case correctness
|
||||
- When writing or modifying tests, check existing test cases to ensure they have scientific naming and rigorous logic testing, if not compliant, modify test cases to ensure scientific and rigorous testing
|
||||
- **Before committing any changes, run `cargo clippy --all-targets --all-features -- -D warnings` to ensure all code passes Clippy checks**
|
||||
- After each development completion, first git add . then git commit -m "feat: feature description" or "fix: issue description", ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
|
||||
- **Keep commit messages concise and under 72 characters** for the title line, use body for detailed explanations if needed
|
||||
- After each development completion, first git push to remote repository
|
||||
- After each change completion, summarize the changes, do not create summary files, provide a brief change description, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
|
||||
- Provide change descriptions needed for PR in the conversation, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)
|
||||
- **Always provide PR descriptions in English** after completing any changes, including:
|
||||
- Clear and concise title following Conventional Commits format
|
||||
- Detailed description of what was changed and why
|
||||
- List of key changes and improvements
|
||||
- Any breaking changes or migration notes if applicable
|
||||
- Testing information and verification steps
|
||||
- **Provide PR descriptions in copyable markdown format** enclosed in code blocks for easy one-click copying
|
||||
|
||||
## 🚫 AI Documentation Generation Restrictions
|
||||
|
||||
### Forbidden Summary Documents
|
||||
|
||||
- **Strictly forbidden to create any form of AI-generated summary documents**
|
||||
- **Do not create documents containing large amounts of emoji, detailed formatting tables and typical AI style**
|
||||
- **Do not generate the following types of documents in the project:**
|
||||
- Benchmark summary documents (BENCHMARK*.md)
|
||||
- Implementation comparison analysis documents (IMPLEMENTATION_COMPARISON*.md)
|
||||
- Performance analysis report documents
|
||||
- Architecture summary documents
|
||||
- Feature comparison documents
|
||||
- Any documents with large amounts of emoji and formatted content
|
||||
- **If documentation is needed, only create when explicitly requested by the user, and maintain a concise and practical style**
|
||||
- **Documentation should focus on actually needed information, avoiding excessive formatting and decorative content**
|
||||
- **Any discovered AI-generated summary documents should be immediately deleted**
|
||||
|
||||
### Allowed Documentation Types
|
||||
|
||||
- README.md (project introduction, keep concise)
|
||||
- Technical documentation (only create when explicitly needed)
|
||||
- User manual (only create when explicitly needed)
|
||||
- API documentation (generated from code)
|
||||
- Changelog (CHANGELOG.md)
|
||||
|
||||
These rules should serve as guiding principles when developing the RustFS project, ensuring code quality, performance, and maintainability.
|
||||
@@ -0,0 +1,68 @@
|
||||
# Claude AI Rules for RustFS Project
|
||||
|
||||
## Core Rules Reference
|
||||
|
||||
This project follows the comprehensive AI coding rules defined in `.rules.md`. Please refer to that file for the complete set of development guidelines, coding standards, and best practices.
|
||||
|
||||
## Claude-Specific Configuration
|
||||
|
||||
When using Claude for this project, ensure you:
|
||||
|
||||
1. **Review the unified rules**: Always check `.rules.md` for the latest project guidelines
|
||||
2. **Follow branch protection**: Never attempt to commit directly to main/master branch
|
||||
3. **Use English**: All code comments, documentation, and variable names must be in English
|
||||
4. **Clean code practices**: Only make modifications you're confident about
|
||||
5. **Test thoroughly**: Ensure all changes pass formatting, linting, and testing requirements
|
||||
6. **Clean up after yourself**: Remove any temporary scripts or test files created during the session
|
||||
|
||||
## Quick Reference
|
||||
|
||||
### Critical Rules
|
||||
- 🚫 **NEVER commit directly to main/master branch**
|
||||
- ✅ **ALWAYS work on feature branches**
|
||||
- 📝 **ALWAYS use English for code and documentation**
|
||||
- 🧹 **ALWAYS clean up temporary files after use**
|
||||
- 🎯 **ONLY make confident, necessary modifications**
|
||||
|
||||
### Pre-commit Checklist
|
||||
```bash
|
||||
# Before committing, always run:
|
||||
cargo fmt --all
|
||||
cargo clippy --all-targets --all-features -- -D warnings
|
||||
cargo check --all-targets
|
||||
cargo test
|
||||
```
|
||||
|
||||
### Branch Workflow
|
||||
```bash
|
||||
git checkout main
|
||||
git pull origin main
|
||||
git checkout -b feat/your-feature-name
|
||||
# Make your changes
|
||||
git add .
|
||||
git commit -m "feat: your feature description"
|
||||
git push origin feat/your-feature-name
|
||||
gh pr create
|
||||
```
|
||||
|
||||
## Claude-Specific Best Practices
|
||||
|
||||
1. **Task Analysis**: Always thoroughly analyze the task before starting implementation
|
||||
2. **Minimal Changes**: Make only the necessary changes to accomplish the task
|
||||
3. **Clear Communication**: Provide clear explanations of changes and their rationale
|
||||
4. **Error Prevention**: Verify code correctness before suggesting changes
|
||||
5. **Documentation**: Ensure all code changes are properly documented in English
|
||||
|
||||
## Important Notes
|
||||
|
||||
- This file serves as an entry point for Claude AI
|
||||
- All detailed rules and guidelines are maintained in `.rules.md`
|
||||
- Updates to coding standards should be made in `.rules.md` to ensure consistency across all AI tools
|
||||
- When in doubt, always refer to `.rules.md` for authoritative guidance
|
||||
- Claude should prioritize code quality, safety, and maintainability over speed
|
||||
|
||||
## See Also
|
||||
|
||||
- [.rules.md](./.rules.md) - Complete AI coding rules and guidelines
|
||||
- [CONTRIBUTING.md](./CONTRIBUTING.md) - Contribution guidelines
|
||||
- [README.md](./README.md) - Project overview and setup instructions
|
||||
Generated
+163
-265
File diff suppressed because it is too large
Load Diff
+17
-22
@@ -90,32 +90,29 @@ rustfs-checksums = { path = "crates/checksums", version = "0.0.5" }
|
||||
rustfs-workers = { path = "crates/workers", version = "0.0.5" }
|
||||
rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
|
||||
aes-gcm = { version = "0.10.3", features = ["std"] }
|
||||
anyhow = "1.0.98"
|
||||
anyhow = "1.0.99"
|
||||
arc-swap = "1.7.1"
|
||||
argon2 = { version = "0.5.3", features = ["std"] }
|
||||
atoi = "2.0.0"
|
||||
async-channel = "2.5.0"
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.88"
|
||||
async-compression = { version = "0.4.0" }
|
||||
async-compression = { version = "0.4.19" }
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.8.3" }
|
||||
aws-sdk-s3 = "1.100.0"
|
||||
aws-config = { version = "1.8.4" }
|
||||
aws-sdk-s3 = "1.101.0"
|
||||
axum = "0.8.4"
|
||||
axum-extra = "0.10.1"
|
||||
axum-server = { version = "0.7.2", features = ["tls-rustls"] }
|
||||
base64-simd = "0.8.0"
|
||||
base64 = "0.22.1"
|
||||
brotli = "8.0.1"
|
||||
bytes = { version = "1.10.1", features = ["serde"] }
|
||||
bytesize = "2.0.1"
|
||||
byteorder = "1.5.0"
|
||||
bytes-utils = "0.1.4"
|
||||
cfg-if = "1.0.1"
|
||||
crc-fast = "1.3.0"
|
||||
crc-fast = "1.4.0"
|
||||
chacha20poly1305 = { version = "0.10.1" }
|
||||
chrono = { version = "0.4.41", features = ["serde"] }
|
||||
clap = { version = "4.5.42", features = ["derive", "env"] }
|
||||
clap = { version = "4.5.44", features = ["derive", "env"] }
|
||||
const-str = { version = "0.6.4", features = ["std", "proc"] }
|
||||
crc32fast = "1.5.0"
|
||||
criterion = { version = "0.7", features = ["html_reports"] }
|
||||
@@ -132,7 +129,7 @@ form_urlencoded = "1.2.1"
|
||||
futures = "0.3.31"
|
||||
futures-core = "0.3.31"
|
||||
futures-util = "0.3.31"
|
||||
glob = "0.3.2"
|
||||
glob = "0.3.3"
|
||||
hex = "0.4.3"
|
||||
hex-simd = "0.8.0"
|
||||
highway = { version = "1.3.0" }
|
||||
@@ -156,7 +153,6 @@ keyring = { version = "3.6.3", features = [
|
||||
] }
|
||||
lazy_static = "1.5.0"
|
||||
libsystemd = { version = "0.7.2" }
|
||||
lru = "0.16"
|
||||
local-ip-address = "0.6.5"
|
||||
lz4 = "1.28.1"
|
||||
matchit = "0.8.4"
|
||||
@@ -192,7 +188,7 @@ percent-encoding = "2.3.1"
|
||||
pin-project-lite = "0.2.16"
|
||||
prost = "0.14.1"
|
||||
pretty_assertions = "1.4.1"
|
||||
quick-xml = "0.38.0"
|
||||
quick-xml = "0.38.1"
|
||||
rand = "0.9.2"
|
||||
rdkafka = { version = "0.38.0", features = ["tokio"] }
|
||||
reed-solomon-simd = { version = "3.0.1" }
|
||||
@@ -210,7 +206,7 @@ rfd = { version = "0.15.4", default-features = false, features = [
|
||||
"xdg-portal",
|
||||
"tokio",
|
||||
] }
|
||||
rmcp = { version = "0.3.2" }
|
||||
rmcp = { version = "0.5.0" }
|
||||
rmp = "0.8.14"
|
||||
rmp-serde = "1.3.0"
|
||||
rsa = "0.9.8"
|
||||
@@ -236,12 +232,12 @@ snafu = "0.8.6"
|
||||
snap = "1.1.1"
|
||||
socket2 = "0.6.0"
|
||||
strum = { version = "0.27.2", features = ["derive"] }
|
||||
sysinfo = "0.36.1"
|
||||
sysinfo = "0.37.0"
|
||||
sysctl = "0.6.0"
|
||||
tempfile = "3.20.0"
|
||||
temp-env = "0.3.6"
|
||||
test-case = "3.3.1"
|
||||
thiserror = "2.0.12"
|
||||
thiserror = "2.0.14"
|
||||
time = { version = "0.3.41", features = [
|
||||
"std",
|
||||
"parsing",
|
||||
@@ -254,10 +250,10 @@ tokio-rustls = { version = "0.26.2", default-features = false }
|
||||
tokio-stream = { version = "0.1.17" }
|
||||
tokio-tar = "0.3.1"
|
||||
tokio-test = "0.4.4"
|
||||
tokio-util = { version = "0.7.15", features = ["io", "compat"] }
|
||||
tonic = { version = "0.14.0", features = ["gzip"] }
|
||||
tonic-prost = { version = "0.14.0" }
|
||||
tonic-prost-build = { version = "0.14.0" }
|
||||
tokio-util = { version = "0.7.16", features = ["io", "compat"] }
|
||||
tonic = { version = "0.14.1", features = ["gzip"] }
|
||||
tonic-prost = { version = "0.14.1" }
|
||||
tonic-prost-build = { version = "0.14.1" }
|
||||
tower = { version = "0.5.2", features = ["timeout"] }
|
||||
tower-http = { version = "0.6.6", features = ["cors"] }
|
||||
tracing = "0.1.41"
|
||||
@@ -266,11 +262,10 @@ tracing-core = "0.1.34"
|
||||
tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = "0.31.0"
|
||||
tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] }
|
||||
tracing-test = "0.2.5"
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.4"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.17.0", features = [
|
||||
uuid = { version = "1.18.0", features = [
|
||||
"v4",
|
||||
"fast-rng",
|
||||
"macro-diagnostics",
|
||||
@@ -283,7 +278,7 @@ zstd = "0.13.3"
|
||||
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
ignored = ["rustfs", "rust-i18n"]
|
||||
ignored = ["rustfs", "rust-i18n", "rustfs-mcp"]
|
||||
|
||||
[profile.wasm-dev]
|
||||
inherits = "dev"
|
||||
|
||||
+15
-5
@@ -26,13 +26,23 @@ RUN ARCH=$(cat /tmp/arch) && \
|
||||
echo "Unsupported architecture: $TARGETARCH" && exit 1; \
|
||||
fi && \
|
||||
if [ "${RELEASE}" = "latest" ]; then \
|
||||
VERSION="latest"; \
|
||||
# For latest, download from GitHub releases using the -latest suffix
|
||||
PACKAGE_NAME="rustfs-linux-${ARCH}-latest.zip"; \
|
||||
# Use GitHub API to get the latest release URL
|
||||
LATEST_RELEASE_URL=$(curl -s https://api.github.com/repos/rustfs/rustfs/releases/latest | grep -o '"browser_download_url": "[^"]*'"${PACKAGE_NAME}"'"' | cut -d'"' -f4 | head -1); \
|
||||
if [ -z "$LATEST_RELEASE_URL" ]; then \
|
||||
echo "Failed to find latest release for ${PACKAGE_NAME}" >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
DOWNLOAD_URL="$LATEST_RELEASE_URL"; \
|
||||
else \
|
||||
VERSION="v${RELEASE#v}"; \
|
||||
# For specific versions, construct the GitHub release URL directly
|
||||
# RELEASE is the GitHub release tag (e.g., "1.0.0-alpha.42")
|
||||
# VERSION is the version in filename (e.g., "v1.0.0-alpha.42")
|
||||
VERSION="v${RELEASE}"; \
|
||||
PACKAGE_NAME="rustfs-linux-${ARCH}-${VERSION}.zip"; \
|
||||
DOWNLOAD_URL="https://github.com/rustfs/rustfs/releases/download/${RELEASE}/${PACKAGE_NAME}"; \
|
||||
fi && \
|
||||
BASE_URL="https://dl.rustfs.com/artifacts/rustfs/release" && \
|
||||
PACKAGE_NAME="rustfs-linux-${ARCH}-${VERSION}.zip" && \
|
||||
DOWNLOAD_URL="${BASE_URL}/${PACKAGE_NAME}" && \
|
||||
echo "Downloading ${PACKAGE_NAME} from ${DOWNLOAD_URL}" >&2 && \
|
||||
curl -f -L "${DOWNLOAD_URL}" -o rustfs.zip && \
|
||||
unzip rustfs.zip -d /build && \
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
DOCKER_CLI := env("DOCKER_CLI", "docker")
|
||||
IMAGE_NAME := env("IMAGE_NAME", "rustfs:v1.0.0")
|
||||
DOCKERFILE_SOURCE := env("DOCKERFILE_SOURCE", "Dockerfile.source")
|
||||
DOCKERFILE_PRODUCTION := env("DOCKERFILE_PRODUCTION", "Dockerfile")
|
||||
CONTAINER_NAME := env("CONTAINER_NAME", "rustfs-dev")
|
||||
|
||||
[group("📒 Help")]
|
||||
[private]
|
||||
default:
|
||||
@just --list --list-heading $'🦀 RustFS justfile manual page:\n'
|
||||
|
||||
[doc("show help")]
|
||||
[group("📒 Help")]
|
||||
help: default
|
||||
|
||||
[doc("run `cargo fmt` to format codes")]
|
||||
[group("👆 Code Quality")]
|
||||
fmt:
|
||||
@echo "🔧 Formatting code..."
|
||||
cargo fmt --all
|
||||
|
||||
[doc("run `cargo fmt` in check mode")]
|
||||
[group("👆 Code Quality")]
|
||||
fmt-check:
|
||||
@echo "📝 Checking code formatting..."
|
||||
cargo fmt --all --check
|
||||
|
||||
[doc("run `cargo clippy`")]
|
||||
[group("👆 Code Quality")]
|
||||
clippy:
|
||||
@echo "🔍 Running clippy checks..."
|
||||
cargo clippy --all-targets --all-features --fix --allow-dirty -- -D warnings
|
||||
|
||||
[doc("run `cargo check`")]
|
||||
[group("👆 Code Quality")]
|
||||
check:
|
||||
@echo "🔨 Running compilation check..."
|
||||
cargo check --all-targets
|
||||
|
||||
[doc("run `cargo test`")]
|
||||
[group("👆 Code Quality")]
|
||||
test:
|
||||
@echo "🧪 Running tests..."
|
||||
cargo nextest run --all --exclude e2e_test
|
||||
cargo test --all --doc
|
||||
|
||||
[doc("run `fmt` `clippy` `check` `test` at once")]
|
||||
[group("👆 Code Quality")]
|
||||
pre-commit: fmt clippy check test
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
[group("🤔 Git")]
|
||||
setup-hooks:
|
||||
@echo "🔧 Setting up git hooks..."
|
||||
chmod +x .git/hooks/pre-commit
|
||||
@echo "✅ Git hooks setup complete!"
|
||||
|
||||
[doc("use `release` mode for building")]
|
||||
[group("🔨 Build")]
|
||||
build:
|
||||
@echo "🔨 Building RustFS using build-rustfs.sh script..."
|
||||
./build-rustfs.sh
|
||||
|
||||
[doc("use `debug` mode for building")]
|
||||
[group("🔨 Build")]
|
||||
build-dev:
|
||||
@echo "🔨 Building RustFS in development mode..."
|
||||
./build-rustfs.sh --dev
|
||||
|
||||
[group("🔨 Build")]
|
||||
[private]
|
||||
build-target target:
|
||||
@echo "🔨 Building rustfs for {{ target }}..."
|
||||
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
|
||||
./build-rustfs.sh --platform {{ target }}
|
||||
|
||||
[doc("use `x86_64-unknown-linux-musl` target for building")]
|
||||
[group("🔨 Build")]
|
||||
build-musl: (build-target "x86_64-unknown-linux-musl")
|
||||
|
||||
[doc("use `x86_64-unknown-linux-gnu` target for building")]
|
||||
[group("🔨 Build")]
|
||||
build-gnu: (build-target "x86_64-unknown-linux-gnu")
|
||||
|
||||
[doc("use `aarch64-unknown-linux-musl` target for building")]
|
||||
[group("🔨 Build")]
|
||||
build-musl-arm64: (build-target "aarch64-unknown-linux-musl")
|
||||
|
||||
[doc("use `aarch64-unknown-linux-gnu` target for building")]
|
||||
[group("🔨 Build")]
|
||||
build-gnu-arm64: (build-target "aarch64-unknown-linux-gnu")
|
||||
|
||||
[doc("build and deploy to server")]
|
||||
[group("🔨 Build")]
|
||||
deploy-dev ip: build-musl
|
||||
@echo "🚀 Deploying to dev server: {{ ip }}"
|
||||
./scripts/dev_deploy.sh {{ ip }}
|
||||
|
||||
[group("🔨 Build")]
|
||||
[private]
|
||||
build-cross-all-pre:
|
||||
@echo "🔧 Building all target architectures..."
|
||||
@echo "💡 On macOS/Windows, use 'make docker-dev' for reliable multi-arch builds"
|
||||
@echo "🔨 Generating protobuf code..."
|
||||
-cargo run --bin gproto
|
||||
|
||||
[doc("build all targets at once")]
|
||||
[group("🔨 Build")]
|
||||
build-cross-all: build-cross-all-pre && build-gnu build-gnu-arm64 build-musl build-musl-arm64
|
||||
|
||||
# ========================================================================================
|
||||
# Docker Multi-Architecture Builds (Primary Methods)
|
||||
# ========================================================================================
|
||||
|
||||
[doc("build an image and run it")]
|
||||
[group("🐳 Build Image")]
|
||||
build-docker os="rockylinux9.3" cli=(DOCKER_CLI) dockerfile=(DOCKERFILE_SOURCE):
|
||||
#!/usr/bin/env bash
|
||||
SOURCE_BUILD_IMAGE_NAME="rustfs/rustfs-{{ os }}:v1"
|
||||
SOURCE_BUILD_CONTAINER_NAME="rustfs-{{ os }}-build"
|
||||
BUILD_CMD="/root/.cargo/bin/cargo build --release --bin rustfs --target-dir /root/s3-rustfs/target/{{ os }}"
|
||||
echo "🐳 Building RustFS using Docker ({{ os }})..."
|
||||
{{ cli }} buildx build -t $SOURCE_BUILD_IMAGE_NAME -f {{ dockerfile }} .
|
||||
{{ cli }} run --rm --name $SOURCE_BUILD_CONTAINER_NAME -v $(pwd):/root/s3-rustfs -it $SOURCE_BUILD_IMAGE_NAME $BUILD_CMD
|
||||
|
||||
[doc("build an image")]
|
||||
[group("🐳 Build Image")]
|
||||
docker-buildx:
|
||||
@echo "🏗️ Building multi-architecture production Docker images with buildx..."
|
||||
./docker-buildx.sh
|
||||
|
||||
[doc("build an image and push it")]
|
||||
[group("🐳 Build Image")]
|
||||
docker-buildx-push:
|
||||
@echo "🚀 Building and pushing multi-architecture production Docker images with buildx..."
|
||||
./docker-buildx.sh --push
|
||||
|
||||
[doc("build an image with a version")]
|
||||
[group("🐳 Build Image")]
|
||||
docker-buildx-version version:
|
||||
@echo "🏗️ Building multi-architecture production Docker images (version: {{ version }}..."
|
||||
./docker-buildx.sh --release {{ version }}
|
||||
|
||||
[doc("build an image with a version and push it")]
|
||||
[group("🐳 Build Image")]
|
||||
docker-buildx-push-version version:
|
||||
@echo "🚀 Building and pushing multi-architecture production Docker images (version: {{ version }}..."
|
||||
./docker-buildx.sh --release {{ version }} --push
|
||||
|
||||
[doc("build an image with a version and push it to registry")]
|
||||
[group("🐳 Build Image")]
|
||||
docker-dev-push registry cli=(DOCKER_CLI) source=(DOCKERFILE_SOURCE):
|
||||
@echo "🚀 Building and pushing multi-architecture development Docker images..."
|
||||
@echo "💡 push to registry: {{ registry }}"
|
||||
{{ cli }} buildx build \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--file {{ source }} \
|
||||
--tag {{ registry }}/rustfs:source-latest \
|
||||
--tag {{ registry }}/rustfs:dev-latest \
|
||||
--push \
|
||||
.
|
||||
|
||||
# Local production builds using direct buildx (alternative to docker-buildx.sh)
|
||||
|
||||
[group("🐳 Build Image")]
|
||||
docker-buildx-production-local cli=(DOCKER_CLI) source=(DOCKERFILE_PRODUCTION):
|
||||
@echo "🏗️ Building single-architecture production Docker image locally..."
|
||||
@echo "💡 Alternative to docker-buildx.sh for local testing"
|
||||
{{ cli }} buildx build \
|
||||
--file {{ source }} \
|
||||
--tag rustfs:production-latest \
|
||||
--tag rustfs:latest \
|
||||
--load \
|
||||
--build-arg RELEASE=latest \
|
||||
.
|
||||
|
||||
# Development/Source builds using direct buildx commands
|
||||
|
||||
[group("🐳 Build Image")]
|
||||
docker-dev cli=(DOCKER_CLI) source=(DOCKERFILE_SOURCE):
|
||||
@echo "🏗️ Building multi-architecture development Docker images with buildx..."
|
||||
@echo "💡 This builds from source code and is intended for local development and testing"
|
||||
@echo "⚠️ Multi-arch images cannot be loaded locally, use docker-dev-push to push to registry"
|
||||
{{ cli }} buildx build \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--file {{ source }} \
|
||||
--tag rustfs:source-latest \
|
||||
--tag rustfs:dev-latest \
|
||||
.
|
||||
|
||||
[group("🐳 Build Image")]
|
||||
docker-dev-local cli=(DOCKER_CLI) source=(DOCKERFILE_SOURCE):
|
||||
@echo "🏗️ Building single-architecture development Docker image for local use..."
|
||||
@echo "💡 This builds from source code for the current platform and loads locally"
|
||||
{{ cli }} buildx build \
|
||||
--file {{ source }} \
|
||||
--tag rustfs:source-latest \
|
||||
--tag rustfs:dev-latest \
|
||||
--load \
|
||||
.
|
||||
|
||||
# ========================================================================================
|
||||
# Single Architecture Docker Builds (Traditional)
|
||||
# ========================================================================================
|
||||
|
||||
[group("🐳 Build Image")]
|
||||
docker-build-production cli=(DOCKER_CLI) source=(DOCKERFILE_PRODUCTION):
|
||||
@echo "🏗️ Building single-architecture production Docker image..."
|
||||
@echo "💡 Consider using 'make docker-buildx-production-local' for multi-arch support"
|
||||
{{ cli }} build -f {{ source }} -t rustfs:latest .
|
||||
|
||||
[group("🐳 Build Image")]
|
||||
docker-build-source cli=(DOCKER_CLI) source=(DOCKERFILE_SOURCE):
|
||||
@echo "🏗️ Building single-architecture source Docker image..."
|
||||
@echo "💡 Consider using 'make docker-dev-local' for multi-arch support"
|
||||
{{ cli }} build -f {{ source }} -t rustfs:source .
|
||||
|
||||
# ========================================================================================
|
||||
# Development Environment
|
||||
# ========================================================================================
|
||||
|
||||
[group("🏃 Running")]
|
||||
dev-env-start cli=(DOCKER_CLI) source=(DOCKERFILE_SOURCE) container=(CONTAINER_NAME):
|
||||
@echo "🚀 Starting development environment..."
|
||||
{{ cli }} buildx build \
|
||||
--file {{ source }} \
|
||||
--tag rustfs:dev \
|
||||
--load \
|
||||
.
|
||||
-{{ cli }} stop {{ container }} 2>/dev/null
|
||||
-{{ cli }} rm {{ container }} 2>/dev/null
|
||||
{{ cli }} run -d --name {{ container }} \
|
||||
-p 9010:9010 -p 9000:9000 \
|
||||
-v {{ invocation_directory() }}:/workspace \
|
||||
-it rustfs:dev
|
||||
|
||||
[group("🏃 Running")]
|
||||
dev-env-stop cli=(DOCKER_CLI) container=(CONTAINER_NAME):
|
||||
@echo "🛑 Stopping development environment..."
|
||||
-{{ cli }} stop {{ container }} 2>/dev/null
|
||||
-{{ cli }} rm {{ container }} 2>/dev/null
|
||||
|
||||
[group("🏃 Running")]
|
||||
dev-env-restart: dev-env-stop dev-env-start
|
||||
|
||||
[group("👍 E2E")]
|
||||
e2e-server:
|
||||
sh scripts/run.sh
|
||||
|
||||
[group("👍 E2E")]
|
||||
probe-e2e:
|
||||
sh scripts/probe.sh
|
||||
|
||||
[doc("inspect one image")]
|
||||
[group("🚚 Other")]
|
||||
docker-inspect-multiarch image cli=(DOCKER_CLI):
|
||||
@echo "🔍 Inspecting multi-architecture image: {{ image }}"
|
||||
{{ cli }} buildx imagetools inspect {{ image }}
|
||||
@@ -1,5 +1,5 @@
|
||||
###########
|
||||
# 远程开发,需要 VSCode 安装 Dev Containers, Remote SSH, Remote Explorer
|
||||
# Remote development requires VSCode with Dev Containers, Remote SSH, Remote Explorer
|
||||
# https://code.visualstudio.com/docs/remote/containers
|
||||
###########
|
||||
DOCKER_CLI ?= docker
|
||||
@@ -76,7 +76,7 @@ build-docker: SOURCE_BUILD_CONTAINER_NAME = rustfs-$(BUILD_OS)-build
|
||||
build-docker: BUILD_CMD = /root/.cargo/bin/cargo build --release --bin rustfs --target-dir /root/s3-rustfs/target/$(BUILD_OS)
|
||||
build-docker:
|
||||
@echo "🐳 Building RustFS using Docker ($(BUILD_OS))..."
|
||||
$(DOCKER_CLI) build -t $(SOURCE_BUILD_IMAGE_NAME) -f $(DOCKERFILE_SOURCE) .
|
||||
$(DOCKER_CLI) buildx build -t $(SOURCE_BUILD_IMAGE_NAME) -f $(DOCKERFILE_SOURCE) .
|
||||
$(DOCKER_CLI) run --rm --name $(SOURCE_BUILD_CONTAINER_NAME) -v $(shell pwd):/root/s3-rustfs -it $(SOURCE_BUILD_IMAGE_NAME) $(BUILD_CMD)
|
||||
|
||||
.PHONY: build-musl
|
||||
@@ -126,7 +126,7 @@ docker-buildx-push:
|
||||
.PHONY: docker-buildx-version
|
||||
docker-buildx-version:
|
||||
@if [ -z "$(VERSION)" ]; then \
|
||||
echo "❌ 错误: 请指定版本, 例如: make docker-buildx-version VERSION=v1.0.0"; \
|
||||
echo "❌ Error: Please specify version, example: make docker-buildx-version VERSION=v1.0.0"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "🏗️ Building multi-architecture production Docker images (version: $(VERSION))..."
|
||||
@@ -135,7 +135,7 @@ docker-buildx-version:
|
||||
.PHONY: docker-buildx-push-version
|
||||
docker-buildx-push-version:
|
||||
@if [ -z "$(VERSION)" ]; then \
|
||||
echo "❌ 错误: 请指定版本, 例如: make docker-buildx-push-version VERSION=v1.0.0"; \
|
||||
echo "❌ Error: Please specify version, example: make docker-buildx-push-version VERSION=v1.0.0"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "🚀 Building and pushing multi-architecture production Docker images (version: $(VERSION))..."
|
||||
@@ -168,11 +168,11 @@ docker-dev-local:
|
||||
.PHONY: docker-dev-push
|
||||
docker-dev-push:
|
||||
@if [ -z "$(REGISTRY)" ]; then \
|
||||
echo "❌ 错误: 请指定镜像仓库, 例如: make docker-dev-push REGISTRY=ghcr.io/username"; \
|
||||
echo "❌ Error: Please specify registry, example: make docker-dev-push REGISTRY=ghcr.io/username"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "🚀 Building and pushing multi-architecture development Docker images..."
|
||||
@echo "💡 推送到仓库: $(REGISTRY)"
|
||||
@echo "💡 Pushing to registry: $(REGISTRY)"
|
||||
$(DOCKER_CLI) buildx build \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--file $(DOCKERFILE_SOURCE) \
|
||||
@@ -249,7 +249,7 @@ dev-env-restart: dev-env-stop dev-env-start
|
||||
.PHONY: docker-inspect-multiarch
|
||||
docker-inspect-multiarch:
|
||||
@if [ -z "$(IMAGE)" ]; then \
|
||||
echo "❌ 错误: 请指定镜像, 例如: make docker-inspect-multiarch IMAGE=rustfs/rustfs:latest"; \
|
||||
echo "❌ Error: Please specify image, example: make docker-inspect-multiarch IMAGE=rustfs/rustfs:latest"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "🔍 Inspecting multi-architecture image: $(IMAGE)"
|
||||
@@ -277,93 +277,93 @@ build-cross-all:
|
||||
|
||||
.PHONY: help-build
|
||||
help-build:
|
||||
@echo "🔨 RustFS 构建帮助:"
|
||||
@echo "🔨 RustFS Build Help:"
|
||||
@echo ""
|
||||
@echo "🚀 本地构建 (推荐使用):"
|
||||
@echo " make build # 构建 RustFS 二进制文件 (默认包含 console)"
|
||||
@echo " make build-dev # 开发模式构建"
|
||||
@echo " make build-musl # 构建 x86_64 musl 版本"
|
||||
@echo " make build-gnu # 构建 x86_64 GNU 版本"
|
||||
@echo " make build-musl-arm64 # 构建 aarch64 musl 版本"
|
||||
@echo " make build-gnu-arm64 # 构建 aarch64 GNU 版本"
|
||||
@echo "🚀 Local Build (Recommended):"
|
||||
@echo " make build # Build RustFS binary (includes console by default)"
|
||||
@echo " make build-dev # Development mode build"
|
||||
@echo " make build-musl # Build x86_64 musl version"
|
||||
@echo " make build-gnu # Build x86_64 GNU version"
|
||||
@echo " make build-musl-arm64 # Build aarch64 musl version"
|
||||
@echo " make build-gnu-arm64 # Build aarch64 GNU version"
|
||||
@echo ""
|
||||
@echo "🐳 Docker 构建:"
|
||||
@echo " make build-docker # 使用 Docker 容器构建"
|
||||
@echo " make build-docker BUILD_OS=ubuntu22.04 # 指定构建系统"
|
||||
@echo "🐳 Docker Build:"
|
||||
@echo " make build-docker # Build using Docker container"
|
||||
@echo " make build-docker BUILD_OS=ubuntu22.04 # Specify build system"
|
||||
@echo ""
|
||||
@echo "🏗️ 跨架构构建:"
|
||||
@echo " make build-cross-all # 构建所有架构的二进制文件"
|
||||
@echo "🏗️ Cross-architecture Build:"
|
||||
@echo " make build-cross-all # Build binaries for all architectures"
|
||||
@echo ""
|
||||
@echo "🔧 直接使用 build-rustfs.sh 脚本:"
|
||||
@echo " ./build-rustfs.sh --help # 查看脚本帮助"
|
||||
@echo " ./build-rustfs.sh --no-console # 构建时跳过 console 资源"
|
||||
@echo " ./build-rustfs.sh --force-console-update # 强制更新 console 资源"
|
||||
@echo " ./build-rustfs.sh --dev # 开发模式构建"
|
||||
@echo " ./build-rustfs.sh --sign # 签名二进制文件"
|
||||
@echo " ./build-rustfs.sh --platform x86_64-unknown-linux-gnu # 指定目标平台"
|
||||
@echo " ./build-rustfs.sh --skip-verification # 跳过二进制验证"
|
||||
@echo "🔧 Direct usage of build-rustfs.sh script:"
|
||||
@echo " ./build-rustfs.sh --help # View script help"
|
||||
@echo " ./build-rustfs.sh --no-console # Build without console resources"
|
||||
@echo " ./build-rustfs.sh --force-console-update # Force update console resources"
|
||||
@echo " ./build-rustfs.sh --dev # Development mode build"
|
||||
@echo " ./build-rustfs.sh --sign # Sign binary files"
|
||||
@echo " ./build-rustfs.sh --platform x86_64-unknown-linux-gnu # Specify target platform"
|
||||
@echo " ./build-rustfs.sh --skip-verification # Skip binary verification"
|
||||
@echo ""
|
||||
@echo "💡 build-rustfs.sh 脚本提供了更多选项、智能检测和二进制验证功能"
|
||||
@echo "💡 build-rustfs.sh script provides more options, smart detection and binary verification"
|
||||
|
||||
.PHONY: help-docker
|
||||
help-docker:
|
||||
@echo "🐳 Docker 多架构构建帮助:"
|
||||
@echo "🐳 Docker Multi-architecture Build Help:"
|
||||
@echo ""
|
||||
@echo "🚀 生产镜像构建 (推荐使用 docker-buildx.sh):"
|
||||
@echo " make docker-buildx # 构建生产多架构镜像(不推送)"
|
||||
@echo " make docker-buildx-push # 构建并推送生产多架构镜像"
|
||||
@echo " make docker-buildx-version VERSION=v1.0.0 # 构建指定版本"
|
||||
@echo " make docker-buildx-push-version VERSION=v1.0.0 # 构建并推送指定版本"
|
||||
@echo "🚀 Production Image Build (Recommended to use docker-buildx.sh):"
|
||||
@echo " make docker-buildx # Build production multi-arch image (no push)"
|
||||
@echo " make docker-buildx-push # Build and push production multi-arch image"
|
||||
@echo " make docker-buildx-version VERSION=v1.0.0 # Build specific version"
|
||||
@echo " make docker-buildx-push-version VERSION=v1.0.0 # Build and push specific version"
|
||||
@echo ""
|
||||
@echo "🔧 开发/源码镜像构建 (本地开发测试):"
|
||||
@echo " make docker-dev # 构建开发多架构镜像(无法本地加载)"
|
||||
@echo " make docker-dev-local # 构建开发单架构镜像(本地加载)"
|
||||
@echo " make docker-dev-push REGISTRY=xxx # 构建并推送开发镜像"
|
||||
@echo "🔧 Development/Source Image Build (Local development testing):"
|
||||
@echo " make docker-dev # Build dev multi-arch image (cannot load locally)"
|
||||
@echo " make docker-dev-local # Build dev single-arch image (local load)"
|
||||
@echo " make docker-dev-push REGISTRY=xxx # Build and push dev image"
|
||||
@echo ""
|
||||
@echo "🏗️ 本地生产镜像构建 (替代方案):"
|
||||
@echo " make docker-buildx-production-local # 本地构建生产单架构镜像"
|
||||
@echo "🏗️ Local Production Image Build (Alternative):"
|
||||
@echo " make docker-buildx-production-local # Build production single-arch image locally"
|
||||
@echo ""
|
||||
@echo "📦 单架构构建 (传统方式):"
|
||||
@echo " make docker-build-production # 构建单架构生产镜像"
|
||||
@echo " make docker-build-source # 构建单架构源码镜像"
|
||||
@echo "📦 Single-architecture Build (Traditional way):"
|
||||
@echo " make docker-build-production # Build single-arch production image"
|
||||
@echo " make docker-build-source # Build single-arch source image"
|
||||
@echo ""
|
||||
@echo "🚀 开发环境管理:"
|
||||
@echo " make dev-env-start # 启动开发容器环境"
|
||||
@echo " make dev-env-stop # 停止开发容器环境"
|
||||
@echo " make dev-env-restart # 重启开发容器环境"
|
||||
@echo "🚀 Development Environment Management:"
|
||||
@echo " make dev-env-start # Start development container environment"
|
||||
@echo " make dev-env-stop # Stop development container environment"
|
||||
@echo " make dev-env-restart # Restart development container environment"
|
||||
@echo ""
|
||||
@echo "🔧 辅助工具:"
|
||||
@echo " make build-cross-all # 构建所有架构的二进制文件"
|
||||
@echo " make docker-inspect-multiarch IMAGE=xxx # 检查镜像的架构支持"
|
||||
@echo "🔧 Auxiliary Tools:"
|
||||
@echo " make build-cross-all # Build binaries for all architectures"
|
||||
@echo " make docker-inspect-multiarch IMAGE=xxx # Check image architecture support"
|
||||
@echo ""
|
||||
@echo "📋 环境变量:"
|
||||
@echo " REGISTRY 镜像仓库地址 (推送时需要)"
|
||||
@echo " DOCKERHUB_USERNAME Docker Hub 用户名"
|
||||
@echo " DOCKERHUB_TOKEN Docker Hub 访问令牌"
|
||||
@echo " GITHUB_TOKEN GitHub 访问令牌"
|
||||
@echo "📋 Environment Variables:"
|
||||
@echo " REGISTRY Image registry address (required for push)"
|
||||
@echo " DOCKERHUB_USERNAME Docker Hub username"
|
||||
@echo " DOCKERHUB_TOKEN Docker Hub access token"
|
||||
@echo " GITHUB_TOKEN GitHub access token"
|
||||
@echo ""
|
||||
@echo "💡 建议:"
|
||||
@echo " - 生产用途: 使用 docker-buildx* 命令 (基于预编译二进制)"
|
||||
@echo " - 本地开发: 使用 docker-dev* 命令 (从源码构建)"
|
||||
@echo " - 开发环境: 使用 dev-env-* 命令管理开发容器"
|
||||
@echo "💡 Suggestions:"
|
||||
@echo " - Production use: Use docker-buildx* commands (based on precompiled binaries)"
|
||||
@echo " - Local development: Use docker-dev* commands (build from source)"
|
||||
@echo " - Development environment: Use dev-env-* commands to manage dev containers"
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@echo "🦀 RustFS Makefile 帮助:"
|
||||
@echo "🦀 RustFS Makefile Help:"
|
||||
@echo ""
|
||||
@echo "📋 主要命令分类:"
|
||||
@echo " make help-build # 显示构建相关帮助"
|
||||
@echo " make help-docker # 显示 Docker 相关帮助"
|
||||
@echo "📋 Main Command Categories:"
|
||||
@echo " make help-build # Show build-related help"
|
||||
@echo " make help-docker # Show Docker-related help"
|
||||
@echo ""
|
||||
@echo "🔧 代码质量:"
|
||||
@echo " make fmt # 格式化代码"
|
||||
@echo " make clippy # 运行 clippy 检查"
|
||||
@echo " make test # 运行测试"
|
||||
@echo " make pre-commit # 运行所有预提交检查"
|
||||
@echo "🔧 Code Quality:"
|
||||
@echo " make fmt # Format code"
|
||||
@echo " make clippy # Run clippy checks"
|
||||
@echo " make test # Run tests"
|
||||
@echo " make pre-commit # Run all pre-commit checks"
|
||||
@echo ""
|
||||
@echo "🚀 快速开始:"
|
||||
@echo " make build # 构建 RustFS 二进制"
|
||||
@echo " make docker-dev-local # 构建开发 Docker 镜像(本地)"
|
||||
@echo " make dev-env-start # 启动开发环境"
|
||||
@echo "🚀 Quick Start:"
|
||||
@echo " make build # Build RustFS binary"
|
||||
@echo " make docker-dev-local # Build development Docker image (local)"
|
||||
@echo " make dev-env-start # Start development environment"
|
||||
@echo ""
|
||||
@echo "💡 更多帮助请使用 'make help-build' 或 'make help-docker'"
|
||||
@echo "💡 For more help use 'make help-build' or 'make help-docker'"
|
||||
|
||||
@@ -158,7 +158,7 @@ pub fn Home() -> Element {
|
||||
Meta {
|
||||
name: "description",
|
||||
// TODO: translate to english
|
||||
content: "RustFS RustFS 用热门安全的 Rust 语言开发,兼容 S3 协议。适用于 AI/ML 及海量数据存储、大数据、互联网、工业和保密存储等全部场景。近乎免费使用。遵循 Apache 2 协议,支持国产保密设备和系统。",
|
||||
content: "RustFS is developed in the popular and secure Rust language, compatible with S3 protocol. Suitable for all scenarios including AI/ML and massive data storage, big data, internet, industrial and secure storage. Nearly free to use. Follows Apache 2 license, supports domestic security devices and systems.",
|
||||
}
|
||||
div { class: "min-h-screen flex flex-col items-center bg-white",
|
||||
div { class: "absolute top-4 right-6 flex space-x-2",
|
||||
|
||||
@@ -36,7 +36,7 @@ pub fn Navbar() -> Element {
|
||||
pub struct LoadingSpinnerProps {
|
||||
#[props(default = true)]
|
||||
loading: bool,
|
||||
#[props(default = "正在处理中...")]
|
||||
#[props(default = "Processing...")]
|
||||
text: &'static str,
|
||||
}
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ pub fn Setting() -> Element {
|
||||
let config = config.read().clone();
|
||||
spawn(async move {
|
||||
if let Err(e) = service.read().restart(config).await {
|
||||
ServiceManager::show_error(&format!("发送重启命令失败:{e}"));
|
||||
ServiceManager::show_error(&format!("Failed to send restart command: {e}"));
|
||||
}
|
||||
// reset the status when you're done
|
||||
loading.set(false);
|
||||
@@ -209,7 +209,7 @@ pub fn Setting() -> Element {
|
||||
}
|
||||
LoadingSpinner {
|
||||
loading: loading.read().to_owned(),
|
||||
text: "服务处理中...",
|
||||
text: "Service processing...",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,7 +139,7 @@ impl RustFSConfig {
|
||||
if !stored_config.address.is_empty() && stored_config.address != Self::DEFAULT_ADDRESS_VALUE {
|
||||
config.address = stored_config.address;
|
||||
let (host, port) = Self::extract_host_port(config.address.as_str())
|
||||
.ok_or_else(|| format!("无法从地址 '{}' 中提取主机和端口", config.address))?;
|
||||
.ok_or_else(|| format!("Unable to extract host and port from address '{}'", config.address))?;
|
||||
config.host = host.to_string();
|
||||
config.port = port.to_string();
|
||||
}
|
||||
@@ -538,17 +538,17 @@ mod tests {
|
||||
address: "127.0.0.1:9000".to_string(),
|
||||
host: "127.0.0.1".to_string(),
|
||||
port: "9000".to_string(),
|
||||
access_key: "用户名".to_string(),
|
||||
secret_key: "密码 123".to_string(),
|
||||
domain_name: "测试.com".to_string(),
|
||||
volume_name: "/数据/存储".to_string(),
|
||||
access_key: "username".to_string(),
|
||||
secret_key: "password123".to_string(),
|
||||
domain_name: "test.com".to_string(),
|
||||
volume_name: "/data/storage".to_string(),
|
||||
console_address: "127.0.0.1:9001".to_string(),
|
||||
};
|
||||
|
||||
assert_eq!(config.access_key, "用户名");
|
||||
assert_eq!(config.secret_key, "密码 123");
|
||||
assert_eq!(config.domain_name, "测试.com");
|
||||
assert_eq!(config.volume_name, "/数据/存储");
|
||||
assert_eq!(config.access_key, "username");
|
||||
assert_eq!(config.secret_key, "password123");
|
||||
assert_eq!(config.domain_name, "test.com");
|
||||
assert_eq!(config.volume_name, "/data/storage");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -81,7 +81,7 @@ pub enum ServiceCommand {
|
||||
/// success: true,
|
||||
/// start_time: chrono::Local::now(),
|
||||
/// end_time: chrono::Local::now(),
|
||||
/// message: "服务启动成功".to_string(),
|
||||
/// message: "Service started successfully".to_string(),
|
||||
/// };
|
||||
///
|
||||
/// println!("{:?}", result);
|
||||
@@ -175,7 +175,7 @@ impl ServiceManager {
|
||||
/// ```
|
||||
async fn prepare_service() -> Result<PathBuf, Box<dyn Error>> {
|
||||
// get the user directory
|
||||
let home_dir = dirs::home_dir().ok_or("无法获取用户目录")?;
|
||||
let home_dir = dirs::home_dir().ok_or("Unable to get user directory")?;
|
||||
let rustfs_dir = home_dir.join("rustfs");
|
||||
let bin_dir = rustfs_dir.join("bin");
|
||||
let data_dir = rustfs_dir.join("data");
|
||||
@@ -247,23 +247,23 @@ impl ServiceManager {
|
||||
match cmd {
|
||||
ServiceCommand::Start(config) => {
|
||||
if let Err(e) = Self::start_service(&config).await {
|
||||
Self::show_error(&format!("启动服务失败:{e}"));
|
||||
Self::show_error(&format!("Failed to start service: {e}"));
|
||||
}
|
||||
}
|
||||
ServiceCommand::Stop => {
|
||||
if let Err(e) = Self::stop_service().await {
|
||||
Self::show_error(&format!("停止服务失败:{e}"));
|
||||
Self::show_error(&format!("Failed to stop service: {e}"));
|
||||
}
|
||||
}
|
||||
ServiceCommand::Restart(config) => {
|
||||
if Self::check_service_status().await.is_some() {
|
||||
if let Err(e) = Self::stop_service().await {
|
||||
Self::show_error(&format!("重启服务失败:{e}"));
|
||||
Self::show_error(&format!("Failed to restart service: {e}"));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Err(e) = Self::start_service(&config).await {
|
||||
Self::show_error(&format!("重启服务失败:{e}"));
|
||||
Self::show_error(&format!("Failed to restart service: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -295,7 +295,7 @@ impl ServiceManager {
|
||||
async fn start_service(config: &RustFSConfig) -> Result<(), Box<dyn Error>> {
|
||||
// Check if the service is already running
|
||||
if let Some(existing_pid) = Self::check_service_status().await {
|
||||
return Err(format!("服务已经在运行,PID: {existing_pid}").into());
|
||||
return Err(format!("Service is already running, PID: {existing_pid}").into());
|
||||
}
|
||||
|
||||
// Prepare the service program
|
||||
@@ -307,16 +307,16 @@ impl ServiceManager {
|
||||
}
|
||||
|
||||
// Extract the port from the configuration
|
||||
let main_port = Self::extract_port(&config.address).ok_or("无法解析主服务端口")?;
|
||||
let console_port = Self::extract_port(&config.console_address).ok_or("无法解析控制台端口")?;
|
||||
let main_port = Self::extract_port(&config.address).ok_or("Unable to parse main service port")?;
|
||||
let console_port = Self::extract_port(&config.console_address).ok_or("Unable to parse console port")?;
|
||||
|
||||
let host = config.address.split(':').next().ok_or("无法解析主机地址")?;
|
||||
let host = config.address.split(':').next().ok_or("Unable to parse host address")?;
|
||||
|
||||
// Check the port
|
||||
let ports = vec![main_port, console_port];
|
||||
for port in ports {
|
||||
if Self::is_port_in_use(host, port).await {
|
||||
return Err(format!("端口 {port} 已被占用").into());
|
||||
return Err(format!("Port {port} is already in use").into());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -339,12 +339,12 @@ impl ServiceManager {
|
||||
|
||||
// Check if the service started successfully
|
||||
if Self::is_port_in_use(host, main_port).await {
|
||||
Self::show_info(&format!("服务启动成功!进程 ID: {process_pid}"));
|
||||
Self::show_info(&format!("Service started successfully! Process ID: {process_pid}"));
|
||||
|
||||
Ok(())
|
||||
} else {
|
||||
child.kill().await?;
|
||||
Err("服务启动失败".into())
|
||||
Err("Service failed to start".into())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -378,13 +378,13 @@ impl ServiceManager {
|
||||
// Verify that the service is indeed stopped
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
if Self::check_service_status().await.is_some() {
|
||||
return Err("服务停止失败".into());
|
||||
return Err("Service failed to stop".into());
|
||||
}
|
||||
Self::show_info("服务已成功停止");
|
||||
Self::show_info("Service stopped successfully");
|
||||
|
||||
Ok(())
|
||||
} else {
|
||||
Err("服务未运行".into())
|
||||
Err("Service is not running".into())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -411,7 +411,7 @@ impl ServiceManager {
|
||||
/// ```
|
||||
pub(crate) fn show_error(message: &str) {
|
||||
rfd::MessageDialog::new()
|
||||
.set_title("错误")
|
||||
.set_title("Error")
|
||||
.set_description(message)
|
||||
.set_level(rfd::MessageLevel::Error)
|
||||
.show();
|
||||
@@ -426,7 +426,7 @@ impl ServiceManager {
|
||||
/// ```
|
||||
pub(crate) fn show_info(message: &str) {
|
||||
rfd::MessageDialog::new()
|
||||
.set_title("成功")
|
||||
.set_title("Success")
|
||||
.set_description(message)
|
||||
.set_level(rfd::MessageLevel::Info)
|
||||
.show();
|
||||
@@ -475,7 +475,7 @@ impl ServiceManager {
|
||||
self.command_tx.send(ServiceCommand::Start(config.clone())).await?;
|
||||
|
||||
let host = &config.host;
|
||||
let port = config.port.parse::<u16>().expect("无效的端口号");
|
||||
let port = config.port.parse::<u16>().expect("Invalid port number");
|
||||
// wait for the service to actually start
|
||||
let mut retries = 0;
|
||||
while retries < 30 {
|
||||
@@ -486,14 +486,14 @@ impl ServiceManager {
|
||||
success: true,
|
||||
start_time,
|
||||
end_time,
|
||||
message: "服务启动成功".to_string(),
|
||||
message: "Service started successfully".to_string(),
|
||||
});
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
retries += 1;
|
||||
}
|
||||
|
||||
Err("服务启动超时".into())
|
||||
Err("Service start timeout".into())
|
||||
}
|
||||
|
||||
/// Stop the service
|
||||
@@ -537,14 +537,14 @@ impl ServiceManager {
|
||||
success: true,
|
||||
start_time,
|
||||
end_time,
|
||||
message: "服务停止成功".to_string(),
|
||||
message: "Service stopped successfully".to_string(),
|
||||
});
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
retries += 1;
|
||||
}
|
||||
|
||||
Err("服务停止超时".into())
|
||||
Err("Service stop timeout".into())
|
||||
}
|
||||
|
||||
/// Restart the service
|
||||
@@ -590,7 +590,7 @@ impl ServiceManager {
|
||||
self.command_tx.send(ServiceCommand::Restart(config.clone())).await?;
|
||||
|
||||
let host = &config.host;
|
||||
let port = config.port.parse::<u16>().expect("无效的端口号");
|
||||
let port = config.port.parse::<u16>().expect("Invalid port number");
|
||||
|
||||
// wait for the service to restart
|
||||
let mut retries = 0;
|
||||
@@ -602,8 +602,8 @@ impl ServiceManager {
|
||||
Err(e) => {
|
||||
error!("save config error: {}", e);
|
||||
self.command_tx.send(ServiceCommand::Stop).await?;
|
||||
Self::show_error("保存配置失败");
|
||||
return Err("保存配置失败".into());
|
||||
Self::show_error("Failed to save configuration");
|
||||
return Err("Failed to save configuration".into());
|
||||
}
|
||||
}
|
||||
let end_time = chrono::Local::now();
|
||||
@@ -611,13 +611,13 @@ impl ServiceManager {
|
||||
success: true,
|
||||
start_time,
|
||||
end_time,
|
||||
message: "服务重启成功".to_string(),
|
||||
message: "Service restarted successfully".to_string(),
|
||||
});
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
retries += 1;
|
||||
}
|
||||
Err("服务重启超时".into())
|
||||
Err("Service restart timeout".into())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -802,10 +802,10 @@ mod tests {
|
||||
success: true,
|
||||
start_time: chrono::Local::now(),
|
||||
end_time: chrono::Local::now(),
|
||||
message: "操作成功 🎉".to_string(),
|
||||
message: "Operation successful 🎉".to_string(),
|
||||
};
|
||||
|
||||
assert_eq!(result.message, "操作成功 🎉");
|
||||
assert_eq!(result.message, "Operation successful 🎉");
|
||||
assert!(result.success);
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ use tracing_subscriber::util::SubscriberInitExt;
|
||||
/// that rotates log files daily
|
||||
pub fn init_logger() -> WorkerGuard {
|
||||
// configuring rolling logs rolling by day
|
||||
let home_dir = dirs::home_dir().expect("无法获取用户目录");
|
||||
let home_dir = dirs::home_dir().expect("Unable to get user directory");
|
||||
let rustfs_dir = home_dir.join("rustfs");
|
||||
let logs_dir = rustfs_dir.join("logs");
|
||||
let file_appender = RollingFileAppender::builder()
|
||||
|
||||
@@ -24,24 +24,19 @@ tracing = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
time = { workspace = true, features = ["serde"] }
|
||||
uuid = { workspace = true, features = ["v4", "serde"] }
|
||||
anyhow = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
url = { workspace = true }
|
||||
rustfs-lock = { workspace = true }
|
||||
|
||||
s3s = { workspace = true }
|
||||
lazy_static = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
rmp-serde = { workspace = true }
|
||||
tokio-test = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
serial_test = "3.2.0"
|
||||
once_cell = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
walkdir = "2.5.0"
|
||||
tempfile = { workspace = true }
|
||||
|
||||
@@ -133,8 +133,14 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
match self.ecstore.get_object_info(bucket, object, &Default::default()).await {
|
||||
Ok(info) => Ok(Some(info)),
|
||||
Err(e) => {
|
||||
error!("Failed to get object meta: {}/{} - {}", bucket, object, e);
|
||||
Err(Error::other(e))
|
||||
// Map ObjectNotFound to None to align with Option return type
|
||||
if matches!(e, rustfs_ecstore::error::StorageError::ObjectNotFound(_, _)) {
|
||||
debug!("Object meta not found: {}/{}", bucket, object);
|
||||
Ok(None)
|
||||
} else {
|
||||
error!("Failed to get object meta: {}/{} - {}", bucket, object, e);
|
||||
Err(Error::other(e))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -142,22 +148,47 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
async fn get_object_data(&self, bucket: &str, object: &str) -> Result<Option<Vec<u8>>> {
|
||||
debug!("Getting object data: {}/{}", bucket, object);
|
||||
|
||||
match (*self.ecstore)
|
||||
let reader = match (*self.ecstore)
|
||||
.get_object_reader(bucket, object, None, Default::default(), &Default::default())
|
||||
.await
|
||||
{
|
||||
Ok(mut reader) => match reader.read_all().await {
|
||||
Ok(data) => Ok(Some(data)),
|
||||
Err(e) => {
|
||||
error!("Failed to read object data: {}/{} - {}", bucket, object, e);
|
||||
Err(Error::other(e))
|
||||
}
|
||||
},
|
||||
Ok(reader) => reader,
|
||||
Err(e) => {
|
||||
error!("Failed to get object: {}/{} - {}", bucket, object, e);
|
||||
Err(Error::other(e))
|
||||
return Err(Error::other(e));
|
||||
}
|
||||
};
|
||||
|
||||
// WARNING: Returning Vec<u8> for large objects is dangerous. To avoid OOM, cap the read size.
|
||||
// If needed, refactor callers to stream instead of buffering entire object.
|
||||
const MAX_READ_BYTES: usize = 16 * 1024 * 1024; // 16 MiB cap
|
||||
let mut buf = Vec::with_capacity(1024 * 1024);
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
let mut n_read: usize = 0;
|
||||
let mut stream = reader.stream;
|
||||
loop {
|
||||
// Read in chunks
|
||||
let mut chunk = vec![0u8; 1024 * 1024];
|
||||
match stream.read(&mut chunk).await {
|
||||
Ok(0) => break,
|
||||
Ok(n) => {
|
||||
buf.extend_from_slice(&chunk[..n]);
|
||||
n_read += n;
|
||||
if n_read > MAX_READ_BYTES {
|
||||
warn!(
|
||||
"Object data exceeds cap ({} bytes), aborting full read to prevent OOM: {}/{}",
|
||||
MAX_READ_BYTES, bucket, object
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to read object data: {}/{} - {}", bucket, object, e);
|
||||
return Err(Error::other(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some(buf))
|
||||
}
|
||||
|
||||
async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()> {
|
||||
@@ -197,27 +228,34 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result<bool> {
|
||||
debug!("Verifying object integrity: {}/{}", bucket, object);
|
||||
|
||||
// Try to get object info and data to verify integrity
|
||||
// Check object metadata first
|
||||
match self.get_object_meta(bucket, object).await? {
|
||||
Some(obj_info) => {
|
||||
// Check if object has valid metadata
|
||||
if obj_info.size < 0 {
|
||||
warn!("Object has invalid size: {}/{}", bucket, object);
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Try to read object data to verify it's accessible
|
||||
match self.get_object_data(bucket, object).await {
|
||||
Ok(Some(_)) => {
|
||||
info!("Object integrity check passed: {}/{}", bucket, object);
|
||||
Ok(true)
|
||||
// Stream-read the object to a sink to avoid loading into memory
|
||||
match (*self.ecstore)
|
||||
.get_object_reader(bucket, object, None, Default::default(), &Default::default())
|
||||
.await
|
||||
{
|
||||
Ok(reader) => {
|
||||
let mut stream = reader.stream;
|
||||
match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await {
|
||||
Ok(_) => {
|
||||
info!("Object integrity check passed: {}/{}", bucket, object);
|
||||
Ok(true)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Object stream read failed: {}/{} - {}", bucket, object, e);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
warn!("Object data not found: {}/{}", bucket, object);
|
||||
Ok(false)
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("Object data read failed: {}/{}", bucket, object);
|
||||
Err(e) => {
|
||||
warn!("Failed to get object reader: {}/{} - {}", bucket, object, e);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,22 +23,23 @@ use ecstore::{
|
||||
set_disk::SetDisks,
|
||||
};
|
||||
use rustfs_ecstore::{self as ecstore, StorageAPI, data_usage::store_data_usage_in_backend};
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
use rustfs_filemeta::{MetacacheReader, VersionType};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use super::metrics::{BucketMetrics, DiskMetrics, MetricsCollector, ScannerMetrics};
|
||||
use crate::heal::HealManager;
|
||||
use crate::scanner::lifecycle::ScannerItem;
|
||||
use crate::{
|
||||
HealRequest,
|
||||
error::{Error, Result},
|
||||
get_ahm_services_cancel_token,
|
||||
};
|
||||
use rustfs_common::{
|
||||
data_usage::DataUsageInfo,
|
||||
metrics::{Metric, Metrics, globalMetrics},
|
||||
};
|
||||
|
||||
use rustfs_common::data_usage::DataUsageInfo;
|
||||
use rustfs_common::metrics::{Metric, Metrics, globalMetrics};
|
||||
use rustfs_ecstore::cmd::bucket_targets::VersioningConfig;
|
||||
|
||||
use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
||||
|
||||
@@ -290,7 +291,7 @@ impl Scanner {
|
||||
|
||||
/// Get global metrics from common crate
|
||||
pub async fn get_global_metrics(&self) -> rustfs_madmin::metrics::ScannerMetrics {
|
||||
globalMetrics.report().await
|
||||
(*globalMetrics).report().await
|
||||
}
|
||||
|
||||
/// Perform a single scan cycle
|
||||
@@ -317,7 +318,7 @@ impl Scanner {
|
||||
cycle_completed: vec![chrono::Utc::now()],
|
||||
started: chrono::Utc::now(),
|
||||
};
|
||||
globalMetrics.set_cycle(Some(cycle_info)).await;
|
||||
(*globalMetrics).set_cycle(Some(cycle_info)).await;
|
||||
|
||||
self.metrics.set_current_cycle(self.state.read().await.current_cycle);
|
||||
self.metrics.increment_total_cycles();
|
||||
@@ -431,8 +432,27 @@ impl Scanner {
|
||||
}
|
||||
|
||||
if let Some(ecstore) = rustfs_ecstore::new_object_layer_fn() {
|
||||
// First try the standard integrity check
|
||||
// First check whether the object still logically exists.
|
||||
// If it's already deleted (e.g., non-versioned bucket), do not trigger heal.
|
||||
let object_opts = ecstore::store_api::ObjectOptions::default();
|
||||
match ecstore.get_object_info(bucket, object, &object_opts).await {
|
||||
Ok(_) => {
|
||||
// Object exists logically, continue with verification below
|
||||
}
|
||||
Err(e) => {
|
||||
if matches!(e, ecstore::error::StorageError::ObjectNotFound(_, _)) {
|
||||
debug!(
|
||||
"Object {}/{} not found logically (likely deleted), skip integrity check & heal",
|
||||
bucket, object
|
||||
);
|
||||
return Ok(());
|
||||
} else {
|
||||
debug!("get_object_info error for {}/{}: {}", bucket, object, e);
|
||||
// Fall through to existing logic which will handle accordingly
|
||||
}
|
||||
}
|
||||
}
|
||||
// First try the standard integrity check
|
||||
let mut integrity_failed = false;
|
||||
|
||||
debug!("Running standard object verification for {}/{}", bucket, object);
|
||||
@@ -1160,6 +1180,19 @@ impl Scanner {
|
||||
/// This method collects all objects from a disk for a specific bucket.
|
||||
/// It returns a map of object names to their metadata for later analysis.
|
||||
async fn scan_volume(&self, disk: &DiskStore, bucket: &str) -> Result<HashMap<String, rustfs_filemeta::FileMeta>> {
|
||||
let ecstore = match rustfs_ecstore::new_object_layer_fn() {
|
||||
Some(ecstore) => ecstore,
|
||||
None => {
|
||||
error!("ECStore not available");
|
||||
return Err(Error::Other("ECStore not available".to_string()));
|
||||
}
|
||||
};
|
||||
let bucket_info = ecstore.get_bucket_info(bucket, &Default::default()).await.ok();
|
||||
let versioning_config = bucket_info.map(|bi| Arc::new(VersioningConfig { enabled: bi.versioning }));
|
||||
let lifecycle_config = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket)
|
||||
.await
|
||||
.ok()
|
||||
.map(|(c, _)| Arc::new(c));
|
||||
// Start global metrics collection for volume scan
|
||||
let stop_fn = Metrics::time(Metric::ScanObject);
|
||||
|
||||
@@ -1247,6 +1280,15 @@ impl Scanner {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Apply lifecycle actions
|
||||
if let Some(lifecycle_config) = &lifecycle_config {
|
||||
let mut scanner_item =
|
||||
ScannerItem::new(bucket.to_string(), Some(lifecycle_config.clone()), versioning_config.clone());
|
||||
if let Err(e) = scanner_item.apply_actions(&entry.name, entry.clone()).await {
|
||||
error!("Failed to apply lifecycle actions for {}/{}: {}", bucket, entry.name, e);
|
||||
}
|
||||
}
|
||||
|
||||
// Store object metadata for later analysis
|
||||
object_metadata.insert(entry.name.clone(), file_meta.clone());
|
||||
}
|
||||
@@ -1375,8 +1417,64 @@ impl Scanner {
|
||||
let empty_vec = Vec::new();
|
||||
let locations = object_locations.get(&key).unwrap_or(&empty_vec);
|
||||
|
||||
// If any disk reports this object as a latest delete marker (tombstone),
|
||||
// it's a legitimate deletion. Skip missing-object heal to avoid recreating
|
||||
// deleted objects. Optional: a metadata heal could be submitted to fan-out
|
||||
// the delete marker, but we keep it conservative here.
|
||||
let mut has_latest_delete_marker = false;
|
||||
for &disk_idx in locations {
|
||||
if let Some(bucket_map) = all_disk_objects.get(disk_idx) {
|
||||
if let Some(file_map) = bucket_map.get(bucket) {
|
||||
if let Some(fm) = file_map.get(object_name) {
|
||||
if let Some(first_ver) = fm.versions.first() {
|
||||
if first_ver.header.version_type == VersionType::Delete {
|
||||
has_latest_delete_marker = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if has_latest_delete_marker {
|
||||
debug!(
|
||||
"Object {}/{} is a delete marker on some disk(s), skipping heal for missing parts",
|
||||
bucket, object_name
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if object is missing from some disks
|
||||
if locations.len() < disks.len() {
|
||||
// Before submitting heal, confirm the object still exists logically.
|
||||
let should_heal = if let Some(store) = rustfs_ecstore::new_object_layer_fn() {
|
||||
match store.get_object_info(bucket, object_name, &Default::default()).await {
|
||||
Ok(_) => true, // exists -> propagate by heal
|
||||
Err(e) => {
|
||||
if matches!(e, rustfs_ecstore::error::StorageError::ObjectNotFound(_, _)) {
|
||||
debug!(
|
||||
"Object {}/{} not found logically (deleted), skip missing-disks heal",
|
||||
bucket, object_name
|
||||
);
|
||||
false
|
||||
} else {
|
||||
debug!(
|
||||
"Object {}/{} get_object_info errored ({}), conservatively skip heal",
|
||||
bucket, object_name, e
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No store available; be conservative and skip to avoid recreating deletions
|
||||
debug!("No ECStore available to confirm existence, skip heal for {}/{}", bucket, object_name);
|
||||
false
|
||||
};
|
||||
|
||||
if !should_heal {
|
||||
continue;
|
||||
}
|
||||
objects_needing_heal += 1;
|
||||
let missing_disks: Vec<usize> = (0..disks.len()).filter(|&i| !locations.contains(&i)).collect();
|
||||
warn!("Object {}/{} missing from disks: {:?}", bucket, object_name, missing_disks);
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_common::metrics::IlmAction;
|
||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
|
||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{apply_lifecycle_action, eval_action_from_lifecycle};
|
||||
use rustfs_ecstore::bucket::metadata_sys::get_object_lock_config;
|
||||
use rustfs_ecstore::cmd::bucket_targets::VersioningConfig;
|
||||
use rustfs_ecstore::store_api::ObjectInfo;
|
||||
use rustfs_filemeta::FileMetaVersion;
|
||||
use rustfs_filemeta::metacache::MetaCacheEntry;
|
||||
use s3s::dto::BucketLifecycleConfiguration as LifecycleConfig;
|
||||
use tracing::info;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ScannerItem {
|
||||
bucket: String,
|
||||
lifecycle: Option<Arc<LifecycleConfig>>,
|
||||
versioning: Option<Arc<VersioningConfig>>,
|
||||
}
|
||||
|
||||
impl ScannerItem {
|
||||
pub fn new(bucket: String, lifecycle: Option<Arc<LifecycleConfig>>, versioning: Option<Arc<VersioningConfig>>) -> Self {
|
||||
Self {
|
||||
bucket,
|
||||
lifecycle,
|
||||
versioning,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn apply_actions(&mut self, object: &str, mut meta: MetaCacheEntry) -> anyhow::Result<()> {
|
||||
info!("apply_actions called for object: {}", object);
|
||||
if self.lifecycle.is_none() {
|
||||
info!("No lifecycle config for object: {}", object);
|
||||
return Ok(());
|
||||
}
|
||||
info!("Lifecycle config exists for object: {}", object);
|
||||
|
||||
let file_meta = match meta.xl_meta() {
|
||||
Ok(meta) => meta,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to get xl_meta for {}: {}", object, e);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
let latest_version = file_meta.versions.first().cloned().unwrap_or_default();
|
||||
let file_meta_version = FileMetaVersion::try_from(latest_version.meta.as_slice()).unwrap_or_default();
|
||||
|
||||
let obj_info = ObjectInfo {
|
||||
bucket: self.bucket.clone(),
|
||||
name: object.to_string(),
|
||||
version_id: latest_version.header.version_id,
|
||||
mod_time: latest_version.header.mod_time,
|
||||
size: file_meta_version.object.as_ref().map_or(0, |o| o.size),
|
||||
user_defined: serde_json::from_slice(file_meta.data.as_slice()).unwrap_or_default(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
self.apply_lifecycle(&obj_info).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn apply_lifecycle(&mut self, oi: &ObjectInfo) -> (IlmAction, i64) {
|
||||
let size = oi.size;
|
||||
if self.lifecycle.is_none() {
|
||||
return (IlmAction::NoneAction, size);
|
||||
}
|
||||
|
||||
let (olcfg, rcfg) = if self.bucket != ".minio.sys" {
|
||||
(
|
||||
get_object_lock_config(&self.bucket).await.ok(),
|
||||
None, // FIXME: replication config
|
||||
)
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
let lc_evt = eval_action_from_lifecycle(
|
||||
self.lifecycle.as_ref().unwrap(),
|
||||
olcfg
|
||||
.as_ref()
|
||||
.and_then(|(c, _)| c.rule.as_ref().and_then(|r| r.default_retention.clone())),
|
||||
rcfg.clone(),
|
||||
oi,
|
||||
)
|
||||
.await;
|
||||
|
||||
info!("lifecycle: {} Initial scan: {}", oi.name, lc_evt.action);
|
||||
|
||||
let mut new_size = size;
|
||||
match lc_evt.action {
|
||||
IlmAction::DeleteVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
|
||||
new_size = 0;
|
||||
}
|
||||
IlmAction::DeleteAction => {
|
||||
if let Some(vcfg) = &self.versioning {
|
||||
if !vcfg.is_enabled() {
|
||||
new_size = 0;
|
||||
}
|
||||
} else {
|
||||
new_size = 0;
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await;
|
||||
(lc_evt.action, new_size)
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
pub mod data_scanner;
|
||||
pub mod histogram;
|
||||
pub mod lifecycle;
|
||||
pub mod metrics;
|
||||
|
||||
pub use data_scanner::Scanner;
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
use rustfs_ahm::scanner::{Scanner, data_scanner::ScannerConfig};
|
||||
use rustfs_ecstore::{
|
||||
bucket::metadata::BUCKET_LIFECYCLE_CONFIG,
|
||||
bucket::metadata_sys,
|
||||
disk::endpoint::Endpoint,
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
store::ECStore,
|
||||
store_api::{ObjectIO, ObjectOptions, PutObjReader, StorageAPI},
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::sync::Once;
|
||||
use std::sync::OnceLock;
|
||||
use std::{path::PathBuf, sync::Arc, time::Duration};
|
||||
use tokio::fs;
|
||||
use tracing::info;
|
||||
|
||||
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
fn init_tracing() {
|
||||
INIT.call_once(|| {
|
||||
let _ = tracing_subscriber::fmt::try_init();
|
||||
});
|
||||
}
|
||||
|
||||
/// Test helper: Create test environment with ECStore
|
||||
async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
init_tracing();
|
||||
|
||||
// Fast path: already initialized, just clone and return
|
||||
if let Some((paths, ecstore)) = GLOBAL_ENV.get() {
|
||||
return (paths.clone(), ecstore.clone());
|
||||
}
|
||||
|
||||
// create temp dir as 4 disks with unique base dir
|
||||
let test_base_dir = format!("/tmp/rustfs_ahm_lifecycle_test_{}", uuid::Uuid::new_v4());
|
||||
let temp_dir = std::path::PathBuf::from(&test_base_dir);
|
||||
if temp_dir.exists() {
|
||||
fs::remove_dir_all(&temp_dir).await.ok();
|
||||
}
|
||||
fs::create_dir_all(&temp_dir).await.unwrap();
|
||||
|
||||
// create 4 disk dirs
|
||||
let disk_paths = vec![
|
||||
temp_dir.join("disk1"),
|
||||
temp_dir.join("disk2"),
|
||||
temp_dir.join("disk3"),
|
||||
temp_dir.join("disk4"),
|
||||
];
|
||||
|
||||
for disk_path in &disk_paths {
|
||||
fs::create_dir_all(disk_path).await.unwrap();
|
||||
}
|
||||
|
||||
// create EndpointServerPools
|
||||
let mut endpoints = Vec::new();
|
||||
for (i, disk_path) in disk_paths.iter().enumerate() {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
|
||||
// set correct index
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
|
||||
let pool_endpoints = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "test".to_string(),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
};
|
||||
|
||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||
|
||||
// format disks (only first time)
|
||||
rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
|
||||
let port = 9002; // for simplicity
|
||||
let server_addr: std::net::SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools).await.unwrap();
|
||||
|
||||
// init bucket metadata system
|
||||
let buckets_list = ecstore
|
||||
.list_bucket(&rustfs_ecstore::store_api::BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
// Initialize background expiry workers
|
||||
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
|
||||
// Store in global once lock
|
||||
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
|
||||
|
||||
(disk_paths, ecstore)
|
||||
}
|
||||
|
||||
/// Test helper: Create a test bucket
|
||||
async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
|
||||
(**ecstore)
|
||||
.make_bucket(bucket_name, &Default::default())
|
||||
.await
|
||||
.expect("Failed to create test bucket");
|
||||
info!("Created test bucket: {}", bucket_name);
|
||||
}
|
||||
|
||||
/// Test helper: Upload test object
|
||||
async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) {
|
||||
let mut reader = PutObjReader::from_vec(data.to_vec());
|
||||
let object_info = (**ecstore)
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to upload test object");
|
||||
|
||||
info!("Uploaded test object: {}/{} ({} bytes)", bucket, object, object_info.size);
|
||||
}
|
||||
|
||||
/// Test helper: Set bucket lifecycle configuration
|
||||
async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Create a simple lifecycle configuration XML with 0 days expiry for immediate testing
|
||||
let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>test-rule</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter>
|
||||
<Prefix>test/</Prefix>
|
||||
</Filter>
|
||||
<Expiration>
|
||||
<Days>0</Days>
|
||||
</Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#;
|
||||
|
||||
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test helper: Check if object exists
|
||||
async fn object_exists(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> bool {
|
||||
((**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await).is_ok()
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_lifecycle_expiry_basic() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
|
||||
// Create test bucket and object
|
||||
let bucket_name = "test-lifecycle-bucket";
|
||||
let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
|
||||
let test_data = b"Hello, this is test data for lifecycle expiry!";
|
||||
|
||||
create_test_bucket(&ecstore, bucket_name).await;
|
||||
upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
|
||||
|
||||
// Verify object exists initially
|
||||
assert!(object_exists(&ecstore, bucket_name, object_name).await);
|
||||
println!("✅ Object exists before lifecycle processing");
|
||||
|
||||
// Set lifecycle configuration with very short expiry (0 days = immediate expiry)
|
||||
set_bucket_lifecycle(bucket_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
|
||||
|
||||
// Verify lifecycle configuration was set
|
||||
match rustfs_ecstore::bucket::metadata_sys::get(bucket_name).await {
|
||||
Ok(bucket_meta) => {
|
||||
assert!(bucket_meta.lifecycle_config.is_some());
|
||||
println!("✅ Bucket metadata retrieved successfully");
|
||||
}
|
||||
Err(e) => {
|
||||
println!("❌ Error retrieving bucket metadata: {e:?}");
|
||||
}
|
||||
}
|
||||
|
||||
// Create scanner with very short intervals for testing
|
||||
let scanner_config = ScannerConfig {
|
||||
scan_interval: Duration::from_millis(100),
|
||||
deep_scan_interval: Duration::from_millis(500),
|
||||
max_concurrent_scans: 1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let scanner = Scanner::new(Some(scanner_config), None);
|
||||
|
||||
// Start scanner
|
||||
scanner.start().await.expect("Failed to start scanner");
|
||||
println!("✅ Scanner started");
|
||||
|
||||
// Wait for scanner to process lifecycle rules
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
|
||||
// Manually trigger a scan cycle to ensure lifecycle processing
|
||||
scanner.scan_cycle().await.expect("Failed to trigger scan cycle");
|
||||
println!("✅ Manual scan cycle completed");
|
||||
|
||||
// Wait a bit more for background workers to process expiry tasks
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
|
||||
// Check if object has been expired (deleted)
|
||||
let object_still_exists = object_exists(&ecstore, bucket_name, object_name).await;
|
||||
println!("Object exists after lifecycle processing: {object_still_exists}");
|
||||
|
||||
if object_still_exists {
|
||||
println!("❌ Object was not deleted by lifecycle processing");
|
||||
// Let's try to get object info to see its details
|
||||
match ecstore
|
||||
.get_object_info(bucket_name, object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(obj_info) => {
|
||||
println!(
|
||||
"Object info: name={}, size={}, mod_time={:?}",
|
||||
obj_info.name, obj_info.size, obj_info.mod_time
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
println!("Error getting object info: {e:?}");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
println!("✅ Object was successfully deleted by lifecycle processing");
|
||||
}
|
||||
|
||||
assert!(!object_still_exists);
|
||||
println!("✅ Object successfully expired");
|
||||
|
||||
// Stop scanner
|
||||
let _ = scanner.stop().await;
|
||||
println!("✅ Scanner stopped");
|
||||
|
||||
println!("Lifecycle expiry basic test completed");
|
||||
}
|
||||
@@ -28,18 +28,11 @@ documentation = "https://docs.rs/rustfs-signer/latest/rustfs_checksum/"
|
||||
[dependencies]
|
||||
bytes = { workspace = true }
|
||||
crc-fast = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
http = { workspace = true }
|
||||
http-body = { workspace = true }
|
||||
base64-simd = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
pin-project-lite = { workspace = true }
|
||||
sha1 = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
bytes-utils = { workspace = true }
|
||||
pretty_assertions = { workspace = true }
|
||||
tracing-test = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
@@ -24,6 +24,7 @@ pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1";
|
||||
pub const SHA_256_HEADER_NAME: &str = "x-amz-checksum-sha256";
|
||||
pub const CRC_64_NVME_HEADER_NAME: &str = "x-amz-checksum-crc64nvme";
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
||||
|
||||
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
||||
|
||||
@@ -294,7 +294,7 @@ impl Checksum for Sha256 {
|
||||
Self::size()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default)]
|
||||
struct Md5 {
|
||||
hasher: md5::Md5,
|
||||
|
||||
@@ -19,3 +19,265 @@ pub const ENV_WORD_DELIMITER: &str = "_";
|
||||
/// Medium-drawn lines separator
|
||||
/// This is used to separate words in environment variable names.
|
||||
pub const ENV_WORD_DELIMITER_DASH: &str = "-";
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)]
|
||||
pub enum EnableState {
|
||||
True,
|
||||
False,
|
||||
#[default]
|
||||
Empty,
|
||||
Yes,
|
||||
No,
|
||||
On,
|
||||
Off,
|
||||
Enabled,
|
||||
Disabled,
|
||||
Ok,
|
||||
NotOk,
|
||||
Success,
|
||||
Failure,
|
||||
Active,
|
||||
Inactive,
|
||||
One,
|
||||
Zero,
|
||||
}
|
||||
impl std::fmt::Display for EnableState {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
impl std::str::FromStr for EnableState {
|
||||
type Err = ();
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.trim() {
|
||||
s if s.eq_ignore_ascii_case("true") => Ok(EnableState::True),
|
||||
s if s.eq_ignore_ascii_case("false") => Ok(EnableState::False),
|
||||
"" => Ok(EnableState::Empty),
|
||||
s if s.eq_ignore_ascii_case("yes") => Ok(EnableState::Yes),
|
||||
s if s.eq_ignore_ascii_case("no") => Ok(EnableState::No),
|
||||
s if s.eq_ignore_ascii_case("on") => Ok(EnableState::On),
|
||||
s if s.eq_ignore_ascii_case("off") => Ok(EnableState::Off),
|
||||
s if s.eq_ignore_ascii_case("enabled") => Ok(EnableState::Enabled),
|
||||
s if s.eq_ignore_ascii_case("disabled") => Ok(EnableState::Disabled),
|
||||
s if s.eq_ignore_ascii_case("ok") => Ok(EnableState::Ok),
|
||||
s if s.eq_ignore_ascii_case("not_ok") => Ok(EnableState::NotOk),
|
||||
s if s.eq_ignore_ascii_case("success") => Ok(EnableState::Success),
|
||||
s if s.eq_ignore_ascii_case("failure") => Ok(EnableState::Failure),
|
||||
s if s.eq_ignore_ascii_case("active") => Ok(EnableState::Active),
|
||||
s if s.eq_ignore_ascii_case("inactive") => Ok(EnableState::Inactive),
|
||||
"1" => Ok(EnableState::One),
|
||||
"0" => Ok(EnableState::Zero),
|
||||
_ => Err(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl EnableState {
|
||||
/// Returns the default value for the enum.
|
||||
pub fn get_default() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
/// Returns the string representation of the enum.
|
||||
pub fn as_str(&self) -> &str {
|
||||
match self {
|
||||
EnableState::True => "true",
|
||||
EnableState::False => "false",
|
||||
EnableState::Empty => "",
|
||||
EnableState::Yes => "yes",
|
||||
EnableState::No => "no",
|
||||
EnableState::On => "on",
|
||||
EnableState::Off => "off",
|
||||
EnableState::Enabled => "enabled",
|
||||
EnableState::Disabled => "disabled",
|
||||
EnableState::Ok => "ok",
|
||||
EnableState::NotOk => "not_ok",
|
||||
EnableState::Success => "success",
|
||||
EnableState::Failure => "failure",
|
||||
EnableState::Active => "active",
|
||||
EnableState::Inactive => "inactive",
|
||||
EnableState::One => "1",
|
||||
EnableState::Zero => "0",
|
||||
}
|
||||
}
|
||||
|
||||
/// is_enabled checks if the state represents an enabled condition.
|
||||
pub fn is_enabled(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
EnableState::True
|
||||
| EnableState::Yes
|
||||
| EnableState::On
|
||||
| EnableState::Enabled
|
||||
| EnableState::Ok
|
||||
| EnableState::Success
|
||||
| EnableState::Active
|
||||
| EnableState::One
|
||||
)
|
||||
}
|
||||
|
||||
/// is_disabled checks if the state represents a disabled condition.
|
||||
pub fn is_disabled(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
EnableState::False
|
||||
| EnableState::No
|
||||
| EnableState::Off
|
||||
| EnableState::Disabled
|
||||
| EnableState::NotOk
|
||||
| EnableState::Failure
|
||||
| EnableState::Inactive
|
||||
| EnableState::Zero
|
||||
| EnableState::Empty
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::str::FromStr;
|
||||
#[test]
|
||||
fn test_enable_state_display_and_fromstr() {
|
||||
let cases = [
|
||||
(EnableState::True, "true"),
|
||||
(EnableState::False, "false"),
|
||||
(EnableState::Empty, ""),
|
||||
(EnableState::Yes, "yes"),
|
||||
(EnableState::No, "no"),
|
||||
(EnableState::On, "on"),
|
||||
(EnableState::Off, "off"),
|
||||
(EnableState::Enabled, "enabled"),
|
||||
(EnableState::Disabled, "disabled"),
|
||||
(EnableState::Ok, "ok"),
|
||||
(EnableState::NotOk, "not_ok"),
|
||||
(EnableState::Success, "success"),
|
||||
(EnableState::Failure, "failure"),
|
||||
(EnableState::Active, "active"),
|
||||
(EnableState::Inactive, "inactive"),
|
||||
(EnableState::One, "1"),
|
||||
(EnableState::Zero, "0"),
|
||||
];
|
||||
for (variant, string) in cases.iter() {
|
||||
assert_eq!(&variant.to_string(), string);
|
||||
assert_eq!(EnableState::from_str(string).unwrap(), *variant);
|
||||
}
|
||||
// Test invalid string
|
||||
assert!(EnableState::from_str("invalid").is_err());
|
||||
}
|
||||
#[test]
|
||||
fn test_enable_state_enum() {
|
||||
let cases = [
|
||||
(EnableState::True, "true"),
|
||||
(EnableState::False, "false"),
|
||||
(EnableState::Empty, ""),
|
||||
(EnableState::Yes, "yes"),
|
||||
(EnableState::No, "no"),
|
||||
(EnableState::On, "on"),
|
||||
(EnableState::Off, "off"),
|
||||
(EnableState::Enabled, "enabled"),
|
||||
(EnableState::Disabled, "disabled"),
|
||||
(EnableState::Ok, "ok"),
|
||||
(EnableState::NotOk, "not_ok"),
|
||||
(EnableState::Success, "success"),
|
||||
(EnableState::Failure, "failure"),
|
||||
(EnableState::Active, "active"),
|
||||
(EnableState::Inactive, "inactive"),
|
||||
(EnableState::One, "1"),
|
||||
(EnableState::Zero, "0"),
|
||||
];
|
||||
for (variant, string) in cases.iter() {
|
||||
assert_eq!(variant.to_string(), *string);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_enable_state_enum_from_str() {
|
||||
let cases = [
|
||||
("true", EnableState::True),
|
||||
("false", EnableState::False),
|
||||
("", EnableState::Empty),
|
||||
("yes", EnableState::Yes),
|
||||
("no", EnableState::No),
|
||||
("on", EnableState::On),
|
||||
("off", EnableState::Off),
|
||||
("enabled", EnableState::Enabled),
|
||||
("disabled", EnableState::Disabled),
|
||||
("ok", EnableState::Ok),
|
||||
("not_ok", EnableState::NotOk),
|
||||
("success", EnableState::Success),
|
||||
("failure", EnableState::Failure),
|
||||
("active", EnableState::Active),
|
||||
("inactive", EnableState::Inactive),
|
||||
("1", EnableState::One),
|
||||
("0", EnableState::Zero),
|
||||
];
|
||||
for (string, variant) in cases.iter() {
|
||||
assert_eq!(EnableState::from_str(string).unwrap(), *variant);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_enable_state_default() {
|
||||
let default_state = EnableState::get_default();
|
||||
assert_eq!(default_state, EnableState::Empty);
|
||||
assert_eq!(default_state.as_str(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_enable_state_as_str() {
|
||||
let cases = [
|
||||
(EnableState::True, "true"),
|
||||
(EnableState::False, "false"),
|
||||
(EnableState::Empty, ""),
|
||||
(EnableState::Yes, "yes"),
|
||||
(EnableState::No, "no"),
|
||||
(EnableState::On, "on"),
|
||||
(EnableState::Off, "off"),
|
||||
(EnableState::Enabled, "enabled"),
|
||||
(EnableState::Disabled, "disabled"),
|
||||
(EnableState::Ok, "ok"),
|
||||
(EnableState::NotOk, "not_ok"),
|
||||
(EnableState::Success, "success"),
|
||||
(EnableState::Failure, "failure"),
|
||||
(EnableState::Active, "active"),
|
||||
(EnableState::Inactive, "inactive"),
|
||||
(EnableState::One, "1"),
|
||||
(EnableState::Zero, "0"),
|
||||
];
|
||||
for (variant, string) in cases.iter() {
|
||||
assert_eq!(variant.as_str(), *string);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_enable_state_is_enabled() {
|
||||
let enabled_states = [
|
||||
EnableState::True,
|
||||
EnableState::Yes,
|
||||
EnableState::On,
|
||||
EnableState::Enabled,
|
||||
EnableState::Ok,
|
||||
EnableState::Success,
|
||||
EnableState::Active,
|
||||
EnableState::One,
|
||||
];
|
||||
for state in enabled_states.iter() {
|
||||
assert!(state.is_enabled());
|
||||
}
|
||||
|
||||
let disabled_states = [
|
||||
EnableState::False,
|
||||
EnableState::No,
|
||||
EnableState::Off,
|
||||
EnableState::Disabled,
|
||||
EnableState::NotOk,
|
||||
EnableState::Failure,
|
||||
EnableState::Inactive,
|
||||
EnableState::Zero,
|
||||
EnableState::Empty,
|
||||
];
|
||||
for state in disabled_states.iter() {
|
||||
assert!(state.is_disabled());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,5 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod app;
|
||||
pub(crate) mod env;
|
||||
pub mod app;
|
||||
pub mod env;
|
||||
pub mod tls;
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub const ENV_TLS_KEYLOG: &str = "RUSTFS_TLS_KEYLOG";
|
||||
@@ -18,6 +18,8 @@ pub mod constants;
|
||||
pub use constants::app::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::env::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::tls::*;
|
||||
#[cfg(feature = "notify")]
|
||||
pub mod notify;
|
||||
#[cfg(feature = "observability")]
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
#![cfg(test)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use aws_config::meta::region::RegionProviderChain;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use bytes::Bytes;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tokio::time::sleep;
|
||||
|
||||
const ENDPOINT: &str = "http://localhost:9000";
|
||||
const ACCESS_KEY: &str = "rustfsadmin";
|
||||
const SECRET_KEY: &str = "rustfsadmin";
|
||||
const BUCKET: &str = "test-basic-bucket";
|
||||
|
||||
async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> {
|
||||
let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1"));
|
||||
let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
|
||||
.region(region_provider)
|
||||
.credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static"))
|
||||
.endpoint_url(ENDPOINT)
|
||||
.load()
|
||||
.await;
|
||||
|
||||
let client = Client::from_conf(
|
||||
aws_sdk_s3::Config::from(&shared_config)
|
||||
.to_builder()
|
||||
.force_path_style(true)
|
||||
.build(),
|
||||
);
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
|
||||
match client.create_bucket().bucket(BUCKET).send().await {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
let error_str = e.to_string();
|
||||
if !error_str.contains("BucketAlreadyOwnedByYou") && !error_str.contains("BucketAlreadyExists") {
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "requires running RustFS server at localhost:9000"]
|
||||
async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error::Error>> {
|
||||
use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
|
||||
use tokio::time::Duration;
|
||||
|
||||
let client = create_aws_s3_client().await?;
|
||||
setup_test_bucket(&client).await?;
|
||||
|
||||
// Upload test object first
|
||||
let test_content = "Test object for lifecycle expiration";
|
||||
let lifecycle_object_key = "lifecycle-test-object.txt";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(lifecycle_object_key)
|
||||
.body(Bytes::from(test_content.as_bytes()).into())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Verify object exists initially
|
||||
let resp = client.get_object().bucket(BUCKET).key(lifecycle_object_key).send().await?;
|
||||
assert!(resp.content_length().unwrap_or(0) > 0);
|
||||
|
||||
// Configure lifecycle rule: expire after current time + 3 seconds
|
||||
let expiration = LifecycleExpiration::builder().days(0).build();
|
||||
let filter = LifecycleRuleFilter::builder().prefix(lifecycle_object_key).build();
|
||||
let rule = LifecycleRule::builder()
|
||||
.id("expire-test-object")
|
||||
.filter(filter)
|
||||
.expiration(expiration)
|
||||
.status(aws_sdk_s3::types::ExpirationStatus::Enabled)
|
||||
.build()?;
|
||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(BUCKET)
|
||||
.lifecycle_configuration(lifecycle)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Verify lifecycle configuration was set
|
||||
let resp = client.get_bucket_lifecycle_configuration().bucket(BUCKET).send().await?;
|
||||
let rules = resp.rules();
|
||||
assert!(rules.iter().any(|r| r.id().unwrap_or("") == "expire-test-object"));
|
||||
|
||||
// Wait for lifecycle processing (scanner runs every 1 second)
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
|
||||
// After lifecycle processing, the object should be deleted by the lifecycle rule
|
||||
let get_result = client.get_object().bucket(BUCKET).key(lifecycle_object_key).send().await;
|
||||
|
||||
match get_result {
|
||||
Ok(_) => {
|
||||
panic!("Expected object to be deleted by lifecycle rule, but it still exists");
|
||||
}
|
||||
Err(e) => {
|
||||
if let Some(service_error) = e.as_service_error() {
|
||||
if service_error.is_no_such_key() {
|
||||
println!("Lifecycle configuration test completed - object was successfully deleted by lifecycle rule");
|
||||
} else {
|
||||
panic!("Expected NoSuchKey error, but got: {e:?}");
|
||||
}
|
||||
} else {
|
||||
panic!("Expected service error, but got: {e:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("Lifecycle configuration test completed.");
|
||||
Ok(())
|
||||
}
|
||||
@@ -38,6 +38,79 @@ fn get_cluster_endpoints() -> Vec<Endpoint> {
|
||||
}]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
#[ignore = "requires running RustFS server at localhost:9000"]
|
||||
async fn test_guard_drop_releases_exclusive_lock_local() -> Result<(), Box<dyn Error>> {
|
||||
// Single local client; no external server required
|
||||
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
|
||||
let ns_lock = NamespaceLock::with_clients("e2e_guard_local".to_string(), vec![client]);
|
||||
|
||||
// Acquire exclusive guard
|
||||
let g1 = ns_lock
|
||||
.lock_guard("guard_exclusive", "owner1", Duration::from_millis(100), Duration::from_secs(5))
|
||||
.await?;
|
||||
assert!(g1.is_some(), "first guard acquisition should succeed");
|
||||
|
||||
// While g1 is alive, second exclusive acquisition should fail
|
||||
let g2 = ns_lock
|
||||
.lock_guard("guard_exclusive", "owner2", Duration::from_millis(50), Duration::from_secs(5))
|
||||
.await?;
|
||||
assert!(g2.is_none(), "second guard acquisition should fail while first is held");
|
||||
|
||||
// Drop first guard to trigger background release
|
||||
drop(g1);
|
||||
// Give the background unlock worker a short moment to process
|
||||
sleep(Duration::from_millis(80)).await;
|
||||
|
||||
// Now acquisition should succeed
|
||||
let g3 = ns_lock
|
||||
.lock_guard("guard_exclusive", "owner2", Duration::from_millis(100), Duration::from_secs(5))
|
||||
.await?;
|
||||
assert!(g3.is_some(), "acquisition should succeed after guard drop releases the lock");
|
||||
drop(g3);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
#[ignore = "requires running RustFS server at localhost:9000"]
|
||||
async fn test_guard_shared_then_write_after_drop() -> Result<(), Box<dyn Error>> {
|
||||
// Two shared read guards should coexist; write should be blocked until they drop
|
||||
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
|
||||
let ns_lock = NamespaceLock::with_clients("e2e_guard_rw".to_string(), vec![client]);
|
||||
|
||||
// Acquire two read guards
|
||||
let r1 = ns_lock
|
||||
.rlock_guard("rw_resource", "reader1", Duration::from_millis(100), Duration::from_secs(5))
|
||||
.await?;
|
||||
let r2 = ns_lock
|
||||
.rlock_guard("rw_resource", "reader2", Duration::from_millis(100), Duration::from_secs(5))
|
||||
.await?;
|
||||
assert!(r1.is_some() && r2.is_some(), "both read guards should be acquired");
|
||||
|
||||
// Attempt write while readers hold the lock should fail
|
||||
let w_fail = ns_lock
|
||||
.lock_guard("rw_resource", "writer", Duration::from_millis(50), Duration::from_secs(5))
|
||||
.await?;
|
||||
assert!(w_fail.is_none(), "write should be blocked when read guards are active");
|
||||
|
||||
// Drop read guards to release
|
||||
drop(r1);
|
||||
drop(r2);
|
||||
sleep(Duration::from_millis(80)).await;
|
||||
|
||||
// Now write should succeed
|
||||
let w_ok = ns_lock
|
||||
.lock_guard("rw_resource", "writer", Duration::from_millis(150), Duration::from_secs(5))
|
||||
.await?;
|
||||
assert!(w_ok.is_some(), "write should succeed after read guards are dropped");
|
||||
drop(w_ok);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
#[ignore = "requires running RustFS server at localhost:9000"]
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod lifecycle;
|
||||
mod lock;
|
||||
mod node_interact_test;
|
||||
mod sql;
|
||||
|
||||
@@ -69,7 +69,6 @@ hmac = { workspace = true }
|
||||
sha1 = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
hex-simd = { workspace = true }
|
||||
path-clean = { workspace = true }
|
||||
tempfile.workspace = true
|
||||
hyper.workspace = true
|
||||
hyper-util.workspace = true
|
||||
@@ -101,6 +100,7 @@ rustfs-rio.workspace = true
|
||||
rustfs-signer.workspace = true
|
||||
rustfs-checksums.workspace = true
|
||||
futures-util.workspace = true
|
||||
async-recursion.workspace = true
|
||||
|
||||
[target.'cfg(not(windows))'.dependencies]
|
||||
nix = { workspace = true }
|
||||
@@ -123,4 +123,4 @@ harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "comparison_benchmark"
|
||||
harness = false
|
||||
harness = false
|
||||
|
||||
@@ -516,7 +516,7 @@ impl TransitionState {
|
||||
if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
|
||||
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
|
||||
error!("Transition to {} failed for {}/{} version:{} with {}",
|
||||
task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.expect("err"), err.to_string());
|
||||
task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(), err.to_string());
|
||||
}
|
||||
} else {
|
||||
let mut ts = TierStats {
|
||||
@@ -743,7 +743,7 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
||||
..Default::default()
|
||||
},
|
||||
//lifecycle_audit_event: lae,
|
||||
version_id: Some(oi.version_id.expect("err").to_string()),
|
||||
version_id: oi.version_id.map(|v| v.to_string()),
|
||||
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
|
||||
mod_time: oi.mod_time,
|
||||
@@ -808,7 +808,7 @@ impl LifecycleOps for ObjectInfo {
|
||||
lifecycle::ObjectOpts {
|
||||
name: self.name.clone(),
|
||||
user_tags: self.user_tags.clone(),
|
||||
version_id: self.version_id.expect("err").to_string(),
|
||||
version_id: self.version_id.map(|v| v.to_string()).unwrap_or_default(),
|
||||
mod_time: self.mod_time,
|
||||
size: self.size as usize,
|
||||
is_latest: self.is_latest,
|
||||
@@ -874,7 +874,11 @@ pub async fn eval_action_from_lifecycle(
|
||||
if lock_enabled && enforce_retention_for_deletion(oi) {
|
||||
//if serverDebugLog {
|
||||
if oi.version_id.is_some() {
|
||||
info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err"));
|
||||
info!(
|
||||
"lifecycle: {} v({}) is locked, not deleting",
|
||||
oi.name,
|
||||
oi.version_id.map(|v| v.to_string()).unwrap_or_default()
|
||||
);
|
||||
} else {
|
||||
info!("lifecycle: {} is locked, not deleting", oi.name);
|
||||
}
|
||||
@@ -928,7 +932,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
};
|
||||
|
||||
if lc_event.action.delete_versioned() {
|
||||
opts.version_id = Some(oi.version_id.expect("err").to_string());
|
||||
opts.version_id = oi.version_id.map(|v| v.to_string());
|
||||
}
|
||||
|
||||
opts.versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await;
|
||||
|
||||
@@ -27,6 +27,7 @@ use std::env;
|
||||
use std::fmt::Display;
|
||||
use time::macros::{datetime, offset};
|
||||
use time::{self, Duration, OffsetDateTime};
|
||||
use tracing::info;
|
||||
|
||||
use crate::bucket::lifecycle::rule::TransitionOps;
|
||||
|
||||
@@ -279,7 +280,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
|
||||
async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event {
|
||||
let mut events = Vec::<Event>::new();
|
||||
info!(
|
||||
"eval_inner: object={}, mod_time={:?}, now={:?}, is_latest={}, delete_marker={}",
|
||||
obj.name, obj.mod_time, now, obj.is_latest, obj.delete_marker
|
||||
);
|
||||
if obj.mod_time.expect("err").unix_timestamp() == 0 {
|
||||
info!("eval_inner: mod_time is 0, returning default event");
|
||||
return Event::default();
|
||||
}
|
||||
|
||||
@@ -418,7 +424,16 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
}
|
||||
}
|
||||
|
||||
if obj.is_latest && !obj.delete_marker {
|
||||
info!(
|
||||
"eval_inner: checking expiration condition - is_latest={}, delete_marker={}, version_id={:?}, condition_met={}",
|
||||
obj.is_latest,
|
||||
obj.delete_marker,
|
||||
obj.version_id,
|
||||
(obj.is_latest || obj.version_id.is_empty()) && !obj.delete_marker
|
||||
);
|
||||
// Allow expiration for latest objects OR non-versioned objects (empty version_id)
|
||||
if (obj.is_latest || obj.version_id.is_empty()) && !obj.delete_marker {
|
||||
info!("eval_inner: entering expiration check");
|
||||
if let Some(ref expiration) = rule.expiration {
|
||||
if let Some(ref date) = expiration.date {
|
||||
let date0 = OffsetDateTime::from(date.clone());
|
||||
@@ -435,22 +450,29 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
});
|
||||
}
|
||||
} else if let Some(days) = expiration.days {
|
||||
if days != 0 {
|
||||
let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days);
|
||||
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
|
||||
let mut event = Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
rule_id: rule.id.clone().expect("err!"),
|
||||
due: Some(expected_expiry),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
};
|
||||
/*if rule.expiration.expect("err!").delete_all.val {
|
||||
event.action = IlmAction::DeleteAllVersionsAction
|
||||
}*/
|
||||
events.push(event);
|
||||
}
|
||||
let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days);
|
||||
info!(
|
||||
"eval_inner: expiration check - days={}, obj_time={:?}, expiry_time={:?}, now={:?}, should_expire={}",
|
||||
days,
|
||||
obj.mod_time.expect("err!"),
|
||||
expected_expiry,
|
||||
now,
|
||||
now.unix_timestamp() > expected_expiry.unix_timestamp()
|
||||
);
|
||||
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
|
||||
info!("eval_inner: object should expire, adding DeleteAction");
|
||||
let mut event = Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
rule_id: rule.id.clone().expect("err!"),
|
||||
due: Some(expected_expiry),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
};
|
||||
/*if rule.expiration.expect("err!").delete_all.val {
|
||||
event.action = IlmAction::DeleteAllVersionsAction
|
||||
}*/
|
||||
events.push(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -598,7 +620,7 @@ impl LifecycleCalculate for Transition {
|
||||
|
||||
pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTime {
|
||||
if days == 0 {
|
||||
return mod_time;
|
||||
return OffsetDateTime::UNIX_EPOCH; // Return epoch time to ensure immediate expiry
|
||||
}
|
||||
let t = mod_time
|
||||
.to_offset(offset!(-0:00:00))
|
||||
|
||||
@@ -54,8 +54,8 @@ pub fn get_object_retention_meta(meta: HashMap<String, String>) -> ObjectLockRet
|
||||
}
|
||||
if let Some(till_str) = till_str {
|
||||
let t = OffsetDateTime::parse(till_str, &format_description::well_known::Iso8601::DEFAULT);
|
||||
if t.is_err() {
|
||||
retain_until_date = Date::from(t.expect("err")); //TODO: utc
|
||||
if let Ok(parsed_time) = t {
|
||||
retain_until_date = Date::from(parsed_time);
|
||||
}
|
||||
}
|
||||
ObjectLockRetention {
|
||||
|
||||
@@ -1897,7 +1897,7 @@ impl ReplicationState {
|
||||
} else if !self.replica_status.is_empty() {
|
||||
self.replica_status.clone()
|
||||
} else {
|
||||
return ReplicationStatusType::Unknown;
|
||||
ReplicationStatusType::Unknown
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -440,6 +440,7 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[async_recursion::async_recursion]
|
||||
pub async fn delete_file(
|
||||
&self,
|
||||
base_path: &PathBuf,
|
||||
@@ -803,13 +804,17 @@ impl LocalDisk {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn scan_dir<W: AsyncWrite + Unpin>(
|
||||
#[async_recursion::async_recursion]
|
||||
async fn scan_dir<W>(
|
||||
&self,
|
||||
current: &mut String,
|
||||
opts: &WalkDirOptions,
|
||||
out: &mut MetacacheWriter<W>,
|
||||
objs_returned: &mut i32,
|
||||
) -> Result<()> {
|
||||
) -> Result<()>
|
||||
where
|
||||
W: AsyncWrite + Unpin + Send,
|
||||
{
|
||||
let forward = {
|
||||
opts.forward_to.as_ref().filter(|v| v.starts_with(&*current)).map(|v| {
|
||||
let forward = v.trim_start_matches(&*current);
|
||||
@@ -953,9 +958,8 @@ impl LocalDisk {
|
||||
let name = path_join_buf(&[current, entry]);
|
||||
|
||||
if !dir_stack.is_empty() {
|
||||
if let Some(pop) = dir_stack.pop() {
|
||||
if let Some(pop) = dir_stack.last().cloned() {
|
||||
if pop < name {
|
||||
//
|
||||
out.write_obj(&MetaCacheEntry {
|
||||
name: pop.clone(),
|
||||
..Default::default()
|
||||
@@ -969,6 +973,7 @@ impl LocalDisk {
|
||||
error!("scan_dir err {:?}", er);
|
||||
}
|
||||
}
|
||||
dir_stack.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,11 +156,12 @@ pub enum StorageError {
|
||||
#[error("Object exists on :{0} as directory {1}")]
|
||||
ObjectExistsAsDirectory(String, String),
|
||||
|
||||
// #[error("Storage resources are insufficient for the read operation")]
|
||||
// InsufficientReadQuorum,
|
||||
#[error("Storage resources are insufficient for the read operation: {0}/{1}")]
|
||||
InsufficientReadQuorum(String, String),
|
||||
|
||||
#[error("Storage resources are insufficient for the write operation: {0}/{1}")]
|
||||
InsufficientWriteQuorum(String, String),
|
||||
|
||||
// #[error("Storage resources are insufficient for the write operation")]
|
||||
// InsufficientWriteQuorum,
|
||||
#[error("Decommission not started")]
|
||||
DecommissionNotStarted,
|
||||
#[error("Decommission already running")]
|
||||
@@ -413,6 +414,8 @@ impl Clone for StorageError {
|
||||
StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles,
|
||||
StorageError::NoHealRequired => StorageError::NoHealRequired,
|
||||
StorageError::Lock(e) => StorageError::Lock(e.clone()),
|
||||
StorageError::InsufficientReadQuorum(a, b) => StorageError::InsufficientReadQuorum(a.clone(), b.clone()),
|
||||
StorageError::InsufficientWriteQuorum(a, b) => StorageError::InsufficientWriteQuorum(a.clone(), b.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -476,6 +479,8 @@ impl StorageError {
|
||||
StorageError::TooManyOpenFiles => 0x36,
|
||||
StorageError::NoHealRequired => 0x37,
|
||||
StorageError::Lock(_) => 0x38,
|
||||
StorageError::InsufficientReadQuorum(_, _) => 0x39,
|
||||
StorageError::InsufficientWriteQuorum(_, _) => 0x3A,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -541,6 +546,8 @@ impl StorageError {
|
||||
0x36 => Some(StorageError::TooManyOpenFiles),
|
||||
0x37 => Some(StorageError::NoHealRequired),
|
||||
0x38 => Some(StorageError::Lock(rustfs_lock::LockError::internal("Generic lock error".to_string()))),
|
||||
0x39 => Some(StorageError::InsufficientReadQuorum(Default::default(), Default::default())),
|
||||
0x3A => Some(StorageError::InsufficientWriteQuorum(Default::default(), Default::default())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -753,6 +760,17 @@ pub fn to_object_err(err: Error, params: Vec<&str>) -> Error {
|
||||
StorageError::PrefixAccessDenied(bucket, object)
|
||||
}
|
||||
|
||||
StorageError::ErasureReadQuorum => {
|
||||
let bucket = params.first().cloned().unwrap_or_default().to_owned();
|
||||
let object = params.get(1).cloned().map(decode_dir_object).unwrap_or_default();
|
||||
StorageError::InsufficientReadQuorum(bucket, object)
|
||||
}
|
||||
StorageError::ErasureWriteQuorum => {
|
||||
let bucket = params.first().cloned().unwrap_or_default().to_owned();
|
||||
let object = params.get(1).cloned().map(decode_dir_object).unwrap_or_default();
|
||||
StorageError::InsufficientWriteQuorum(bucket, object)
|
||||
}
|
||||
|
||||
_ => err,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,15 +177,17 @@ impl S3PeerSys {
|
||||
let pools = cli.get_pools();
|
||||
let idx = i;
|
||||
if pools.unwrap_or_default().contains(&idx) {
|
||||
per_pool_errs.push(errors[j].as_ref());
|
||||
per_pool_errs.push(errors[j].clone());
|
||||
}
|
||||
|
||||
// TODO: reduceWriteQuorumErrs
|
||||
if let Some(pool_err) =
|
||||
reduce_write_quorum_errs(&per_pool_errs, BUCKET_OP_IGNORED_ERRS, (per_pool_errs.len() / 2) + 1)
|
||||
{
|
||||
return Err(pool_err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO:
|
||||
|
||||
Ok(())
|
||||
}
|
||||
pub async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
|
||||
@@ -387,7 +389,6 @@ impl PeerS3Client for LocalPeerS3Client {
|
||||
if opts.force_create && matches!(e, Error::VolumeExists) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
@@ -405,7 +406,9 @@ impl PeerS3Client for LocalPeerS3Client {
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: reduceWriteQuorumErrs
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, (local_disks.len() / 2) + 1) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+327
-57
@@ -17,6 +17,8 @@
|
||||
|
||||
use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::client::{object_api_utils::extract_etag, transition_api::ReaderImpl};
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
|
||||
@@ -2027,6 +2029,24 @@ impl SetDisks {
|
||||
|
||||
Ok((fi, parts_metadata, op_online_disks))
|
||||
}
|
||||
async fn get_object_info_and_quorum(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<(ObjectInfo, usize)> {
|
||||
let (fi, _, _) = self.get_object_fileinfo(bucket, object, opts, false).await?;
|
||||
|
||||
let write_quorum = fi.write_quorum(self.default_write_quorum());
|
||||
|
||||
let oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
// TODO: replicatio
|
||||
|
||||
if fi.deleted {
|
||||
if opts.version_id.is_none() || opts.delete_marker {
|
||||
return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object]));
|
||||
} else {
|
||||
return Err(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object]));
|
||||
}
|
||||
}
|
||||
|
||||
Ok((oi, write_quorum))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[tracing::instrument(
|
||||
@@ -3211,6 +3231,20 @@ impl ObjectIO for SetDisks {
|
||||
h: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
// Acquire a shared read-lock early to protect read consistency
|
||||
let mut _read_lock_guard: Option<rustfs_lock::LockGuard> = None;
|
||||
if !opts.no_lock {
|
||||
let guard_opt = self
|
||||
.namespace_lock
|
||||
.rlock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?;
|
||||
|
||||
if guard_opt.is_none() {
|
||||
return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
}
|
||||
_read_lock_guard = guard_opt;
|
||||
}
|
||||
|
||||
let (fi, files, disks) = self
|
||||
.get_object_fileinfo(bucket, object, opts, true)
|
||||
.await
|
||||
@@ -3256,7 +3290,10 @@ impl ObjectIO for SetDisks {
|
||||
let object = object.to_owned();
|
||||
let set_index = self.set_index;
|
||||
let pool_index = self.pool_index;
|
||||
// Move the read-lock guard into the task so it lives for the duration of the read
|
||||
let _guard_to_hold = _read_lock_guard; // moved into closure below
|
||||
tokio::spawn(async move {
|
||||
let _guard = _guard_to_hold; // keep guard alive until task ends
|
||||
if let Err(e) = Self::get_object_with_fileinfo(
|
||||
&bucket,
|
||||
&object,
|
||||
@@ -3284,16 +3321,18 @@ impl ObjectIO for SetDisks {
|
||||
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
|
||||
let disks = self.disks.read().await;
|
||||
|
||||
// Acquire per-object exclusive lock via RAII guard. It auto-releases asynchronously on drop.
|
||||
let mut _object_lock_guard: Option<rustfs_lock::LockGuard> = None;
|
||||
if !opts.no_lock {
|
||||
let paths = vec![object.to_string()];
|
||||
let lock_acquired = self
|
||||
let guard_opt = self
|
||||
.namespace_lock
|
||||
.lock_batch(&paths, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.lock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?;
|
||||
|
||||
if !lock_acquired {
|
||||
if guard_opt.is_none() {
|
||||
return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
}
|
||||
_object_lock_guard = guard_opt;
|
||||
}
|
||||
|
||||
let mut user_defined = opts.user_defined.clone();
|
||||
@@ -3500,14 +3539,6 @@ impl ObjectIO for SetDisks {
|
||||
|
||||
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
|
||||
|
||||
// Release lock if it was acquired
|
||||
if !opts.no_lock {
|
||||
let paths = vec![object.to_string()];
|
||||
if let Err(err) = self.namespace_lock.unlock_batch(&paths, &self.locker_owner).await {
|
||||
error!("Failed to unlock object {}: {}", object, err);
|
||||
}
|
||||
}
|
||||
|
||||
for (i, op_disk) in online_disks.iter().enumerate() {
|
||||
if let Some(disk) = op_disk {
|
||||
if disk.is_online().await {
|
||||
@@ -3583,6 +3614,19 @@ impl StorageAPI for SetDisks {
|
||||
return Err(StorageError::NotImplemented);
|
||||
}
|
||||
|
||||
// Guard lock for source object metadata update
|
||||
let mut _lock_guard: Option<rustfs_lock::LockGuard> = None;
|
||||
{
|
||||
let guard_opt = self
|
||||
.namespace_lock
|
||||
.lock_guard(src_object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?;
|
||||
if guard_opt.is_none() {
|
||||
return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
}
|
||||
_lock_guard = guard_opt;
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
let (mut metas, errs) = {
|
||||
@@ -3676,6 +3720,18 @@ impl StorageAPI for SetDisks {
|
||||
}
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
|
||||
// Guard lock for single object delete-version
|
||||
let mut _lock_guard: Option<rustfs_lock::LockGuard> = None;
|
||||
{
|
||||
let guard_opt = self
|
||||
.namespace_lock
|
||||
.lock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?;
|
||||
if guard_opt.is_none() {
|
||||
return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
}
|
||||
_lock_guard = guard_opt;
|
||||
}
|
||||
let disks = self.get_disks(0, 0).await?;
|
||||
let write_quorum = disks.len() / 2 + 1;
|
||||
|
||||
@@ -3732,23 +3788,48 @@ impl StorageAPI for SetDisks {
|
||||
del_errs.push(None)
|
||||
}
|
||||
|
||||
// Per-object guards to keep until function end
|
||||
let mut _guards: HashMap<String, rustfs_lock::LockGuard> = HashMap::new();
|
||||
// Acquire locks for all objects first; mark errors for failures
|
||||
for (i, dobj) in objects.iter().enumerate() {
|
||||
if !_guards.contains_key(&dobj.object_name) {
|
||||
match self
|
||||
.namespace_lock
|
||||
.lock_guard(&dobj.object_name, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?
|
||||
{
|
||||
Some(g) => {
|
||||
_guards.insert(dobj.object_name.clone(), g);
|
||||
}
|
||||
None => {
|
||||
del_errs[i] = Some(Error::other("can not get lock. please retry"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// let mut del_fvers = Vec::with_capacity(objects.len());
|
||||
|
||||
let ver_cfg = BucketVersioningSys::get(bucket).await.unwrap_or_default();
|
||||
|
||||
let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
|
||||
|
||||
for (i, dobj) in objects.iter().enumerate() {
|
||||
let mut vr = FileInfo {
|
||||
name: dobj.object_name.clone(),
|
||||
version_id: dobj.version_id,
|
||||
idx: i,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// 删除
|
||||
del_objects[i].object_name.clone_from(&vr.name);
|
||||
del_objects[i].version_id = vr.version_id.map(|v| v.to_string());
|
||||
vr.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
|
||||
if del_objects[i].version_id.is_none() {
|
||||
let (suspended, versioned) = (opts.version_suspended, opts.versioned);
|
||||
// 删除
|
||||
// del_objects[i].object_name.clone_from(&vr.name);
|
||||
// del_objects[i].version_id = vr.version_id.map(|v| v.to_string());
|
||||
|
||||
if dobj.version_id.is_none() {
|
||||
let (suspended, versioned) = (ver_cfg.suspended(), ver_cfg.prefix_enabled(dobj.object_name.as_str()));
|
||||
if suspended || versioned {
|
||||
vr.mod_time = Some(OffsetDateTime::now_utc());
|
||||
vr.deleted = true;
|
||||
@@ -3788,13 +3869,23 @@ impl StorageAPI for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
vers_map.insert(&dobj.object_name, v);
|
||||
// Only add to vers_map if we hold the lock
|
||||
if _guards.contains_key(&dobj.object_name) {
|
||||
vers_map.insert(&dobj.object_name, v);
|
||||
}
|
||||
}
|
||||
|
||||
let mut vers = Vec::with_capacity(vers_map.len());
|
||||
|
||||
for (_, ver) in vers_map {
|
||||
vers.push(ver);
|
||||
for (_, mut fi_vers) in vers_map {
|
||||
fi_vers.versions.sort_by(|a, b| a.deleted.cmp(&b.deleted));
|
||||
fi_vers.versions.reverse();
|
||||
|
||||
if let Some(index) = fi_vers.versions.iter().position(|fi| fi.deleted) {
|
||||
fi_vers.versions.truncate(index + 1);
|
||||
}
|
||||
|
||||
vers.push(fi_vers);
|
||||
}
|
||||
|
||||
let disks = self.disks.read().await;
|
||||
@@ -3830,6 +3921,18 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
|
||||
// Guard lock for single object delete
|
||||
let mut _lock_guard: Option<rustfs_lock::LockGuard> = None;
|
||||
if !opts.delete_prefix {
|
||||
let guard_opt = self
|
||||
.namespace_lock
|
||||
.lock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?;
|
||||
if guard_opt.is_none() {
|
||||
return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
}
|
||||
_lock_guard = guard_opt;
|
||||
}
|
||||
if opts.delete_prefix {
|
||||
self.delete_prefix(bucket, object)
|
||||
.await
|
||||
@@ -3837,7 +3940,145 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
unimplemented!()
|
||||
|
||||
let (oi, write_quorum) = match self.get_object_info_and_quorum(bucket, object, &opts).await {
|
||||
Ok((oi, wq)) => (oi, wq),
|
||||
Err(e) => {
|
||||
return Err(to_object_err(e, vec![bucket, object]));
|
||||
}
|
||||
};
|
||||
|
||||
let mark_delete = oi.version_id.is_some();
|
||||
|
||||
let mut delete_marker = opts.versioned;
|
||||
|
||||
let mod_time = if let Some(mt) = opts.mod_time {
|
||||
mt
|
||||
} else {
|
||||
OffsetDateTime::now_utc()
|
||||
};
|
||||
|
||||
let find_vid = Uuid::new_v4();
|
||||
|
||||
if mark_delete && (opts.versioned || opts.version_suspended) {
|
||||
if !delete_marker {
|
||||
delete_marker = opts.version_suspended && opts.version_id.is_none();
|
||||
}
|
||||
|
||||
let mut fi = FileInfo {
|
||||
name: object.to_string(),
|
||||
deleted: delete_marker,
|
||||
mark_deleted: mark_delete,
|
||||
mod_time: Some(mod_time),
|
||||
..Default::default() // TODO: replication
|
||||
};
|
||||
|
||||
fi.set_tier_free_version_id(&find_vid.to_string());
|
||||
|
||||
if opts.skip_free_version {
|
||||
fi.set_skip_tier_free_version();
|
||||
}
|
||||
|
||||
fi.version_id = if let Some(vid) = opts.version_id {
|
||||
Some(Uuid::parse_str(vid.as_str())?)
|
||||
} else if opts.versioned {
|
||||
Some(Uuid::new_v4())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
self.delete_object_version(bucket, object, &fi, opts.delete_marker)
|
||||
.await
|
||||
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
|
||||
|
||||
return Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended));
|
||||
}
|
||||
|
||||
let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
|
||||
|
||||
// Create a single object deletion request
|
||||
let mut vr = FileInfo {
|
||||
name: object.to_string(),
|
||||
version_id: opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Handle versioning
|
||||
let (suspended, versioned) = (opts.version_suspended, opts.versioned);
|
||||
if opts.version_id.is_none() && (suspended || versioned) {
|
||||
vr.mod_time = Some(OffsetDateTime::now_utc());
|
||||
vr.deleted = true;
|
||||
if versioned {
|
||||
vr.version_id = Some(Uuid::new_v4());
|
||||
}
|
||||
}
|
||||
|
||||
let vers = vec![FileInfoVersions {
|
||||
name: vr.name.clone(),
|
||||
versions: vec![vr.clone()],
|
||||
..Default::default()
|
||||
}];
|
||||
|
||||
let disks = self.disks.read().await;
|
||||
let disks = disks.clone();
|
||||
let write_quorum = disks.len() / 2 + 1;
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk in disks.iter() {
|
||||
let vers = vers.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.delete_versions(bucket, vers, DeleteOptions::default()).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(disk_errs) => {
|
||||
// Handle errors from disk operations
|
||||
for err in disk_errs.iter().flatten() {
|
||||
warn!("delete_object disk error: {:?}", err);
|
||||
}
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check write quorum
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(to_object_err(err.into(), vec![bucket, object]));
|
||||
}
|
||||
|
||||
// Create result ObjectInfo
|
||||
let result_info = if vr.deleted {
|
||||
ObjectInfo {
|
||||
bucket: bucket.to_string(),
|
||||
name: object.to_string(),
|
||||
delete_marker: true,
|
||||
mod_time: vr.mod_time,
|
||||
version_id: vr.version_id,
|
||||
..Default::default()
|
||||
}
|
||||
} else {
|
||||
ObjectInfo {
|
||||
bucket: bucket.to_string(),
|
||||
name: object.to_string(),
|
||||
version_id: vr.version_id,
|
||||
..Default::default()
|
||||
}
|
||||
};
|
||||
|
||||
Ok(result_info)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
@@ -3869,33 +4110,18 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
|
||||
// let mut _ns = None;
|
||||
// if !opts.no_lock {
|
||||
// let paths = vec![object.to_string()];
|
||||
// let ns_lock = new_nslock(
|
||||
// Arc::clone(&self.ns_mutex),
|
||||
// self.locker_owner.clone(),
|
||||
// bucket.to_string(),
|
||||
// paths,
|
||||
// self.lockers.clone(),
|
||||
// )
|
||||
// .await;
|
||||
// if !ns_lock
|
||||
// .0
|
||||
// .write()
|
||||
// .await
|
||||
// .get_lock(&Options {
|
||||
// timeout: Duration::from_secs(5),
|
||||
// retry_interval: Duration::from_secs(1),
|
||||
// })
|
||||
// .await
|
||||
// .map_err(|err| Error::other(err.to_string()))?
|
||||
// {
|
||||
// return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
// }
|
||||
|
||||
// _ns = Some(ns_lock);
|
||||
// }
|
||||
// Acquire a shared read-lock to protect consistency during info fetch
|
||||
let mut _read_lock_guard: Option<rustfs_lock::LockGuard> = None;
|
||||
if !opts.no_lock {
|
||||
let guard_opt = self
|
||||
.namespace_lock
|
||||
.rlock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?;
|
||||
if guard_opt.is_none() {
|
||||
return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
}
|
||||
_read_lock_guard = guard_opt;
|
||||
}
|
||||
|
||||
let (fi, _, _) = self
|
||||
.get_object_fileinfo(bucket, object, opts, false)
|
||||
@@ -3927,6 +4153,19 @@ impl StorageAPI for SetDisks {
|
||||
async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
|
||||
// TODO: nslock
|
||||
|
||||
// Guard lock for metadata update
|
||||
let mut _lock_guard: Option<rustfs_lock::LockGuard> = None;
|
||||
if !opts.no_lock {
|
||||
let guard_opt = self
|
||||
.namespace_lock
|
||||
.lock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?;
|
||||
if guard_opt.is_none() {
|
||||
return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
}
|
||||
_lock_guard = guard_opt;
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
let (metas, errs) = {
|
||||
@@ -4017,12 +4256,18 @@ impl StorageAPI for SetDisks {
|
||||
}
|
||||
};
|
||||
|
||||
/*if !opts.no_lock {
|
||||
let lk = self.new_ns_lock(bucket, object);
|
||||
let lkctx = lk.get_lock(globalDeleteOperationTimeout)?;
|
||||
//ctx = lkctx.Context()
|
||||
//defer lk.Unlock(lkctx)
|
||||
}*/
|
||||
// Acquire write-lock early; hold for the whole transition operation scope
|
||||
let mut _lock_guard: Option<rustfs_lock::LockGuard> = None;
|
||||
if !opts.no_lock {
|
||||
let guard_opt = self
|
||||
.namespace_lock
|
||||
.lock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?;
|
||||
if guard_opt.is_none() {
|
||||
return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
}
|
||||
_lock_guard = guard_opt;
|
||||
}
|
||||
|
||||
let (mut fi, meta_arr, online_disks) = self.get_object_fileinfo(bucket, object, opts, true).await?;
|
||||
/*if err != nil {
|
||||
@@ -4140,6 +4385,18 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
// Acquire write-lock early for the restore operation
|
||||
let mut _lock_guard: Option<rustfs_lock::LockGuard> = None;
|
||||
if !opts.no_lock {
|
||||
let guard_opt = self
|
||||
.namespace_lock
|
||||
.lock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?;
|
||||
if guard_opt.is_none() {
|
||||
return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
}
|
||||
_lock_guard = guard_opt;
|
||||
}
|
||||
let set_restore_header_fn = async move |oi: &mut ObjectInfo, rerr: Option<Error>| -> Result<()> {
|
||||
if rerr.is_none() {
|
||||
return Ok(());
|
||||
@@ -4213,6 +4470,18 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
|
||||
// Acquire write-lock for tag update (metadata write)
|
||||
let mut _lock_guard: Option<rustfs_lock::LockGuard> = None;
|
||||
if !opts.no_lock {
|
||||
let guard_opt = self
|
||||
.namespace_lock
|
||||
.lock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?;
|
||||
if guard_opt.is_none() {
|
||||
return Err(Error::other("can not get lock. please retry".to_string()));
|
||||
}
|
||||
_lock_guard = guard_opt;
|
||||
}
|
||||
let (mut fi, _, disks) = self.get_object_fileinfo(bucket, object, opts, false).await?;
|
||||
|
||||
fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned());
|
||||
@@ -5175,9 +5444,10 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
let mut get_object_reader =
|
||||
<Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
|
||||
let _ = get_object_reader.read_all().await?;
|
||||
let get_object_reader = <Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
|
||||
// Stream to sink to avoid loading entire object into memory during verification
|
||||
let mut reader = get_object_reader.stream;
|
||||
tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -165,7 +165,13 @@ impl Sets {
|
||||
|
||||
let lock_clients = create_unique_clients(&set_endpoints).await?;
|
||||
|
||||
let namespace_lock = rustfs_lock::NamespaceLock::with_clients(format!("set-{i}"), lock_clients);
|
||||
// Bind lock quorum to EC write quorum for this set: data_shards (+1 if equal to parity) per default_write_quorum()
|
||||
let mut write_quorum = set_drive_count - parity_count;
|
||||
if write_quorum == parity_count {
|
||||
write_quorum += 1;
|
||||
}
|
||||
let namespace_lock =
|
||||
rustfs_lock::NamespaceLock::with_clients_and_quorum(format!("set-{i}"), lock_clients, write_quorum);
|
||||
|
||||
let set_disks = SetDisks::new(
|
||||
Arc::new(namespace_lock),
|
||||
@@ -876,11 +882,15 @@ impl StorageAPI for Sets {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
self.get_disks_by_key(object)
|
||||
.verify_object_integrity(bucket, object, opts)
|
||||
.await
|
||||
let gor = self.get_object_reader(bucket, object, None, HeaderMap::new(), opts).await?;
|
||||
let mut reader = gor.stream;
|
||||
|
||||
// Stream data to sink instead of reading all into memory to prevent OOM
|
||||
tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1221,7 +1221,7 @@ impl StorageAPI for ECStore {
|
||||
}
|
||||
|
||||
if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await {
|
||||
let err = err.into();
|
||||
let err = to_object_err(err.into(), vec![bucket]);
|
||||
if !is_err_bucket_exists(&err) {
|
||||
let _ = self
|
||||
.delete_bucket(
|
||||
@@ -1234,7 +1234,6 @@ impl StorageAPI for ECStore {
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
return Err(err);
|
||||
};
|
||||
|
||||
@@ -2238,9 +2237,10 @@ impl StorageAPI for ECStore {
|
||||
}
|
||||
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
let mut get_object_reader =
|
||||
<Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
|
||||
let _ = get_object_reader.read_all().await?;
|
||||
let get_object_reader = <Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
|
||||
// Stream to sink to avoid loading entire object into memory during verification
|
||||
let mut reader = get_object_reader.stream;
|
||||
tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,6 +310,8 @@ pub struct ObjectOptions {
|
||||
pub replication_request: bool,
|
||||
pub delete_marker: bool,
|
||||
|
||||
pub skip_free_version: bool,
|
||||
|
||||
pub transition: TransitionOptions,
|
||||
pub expiration: ExpirationOptions,
|
||||
pub lifecycle_audit_event: LcAuditEvent,
|
||||
|
||||
@@ -139,8 +139,8 @@ async fn init_format_erasure(
|
||||
let idx = i * set_drive_count + j;
|
||||
let mut newfm = fm.clone();
|
||||
newfm.erasure.this = fm.erasure.sets[i][j];
|
||||
if deployment_id.is_some() {
|
||||
newfm.id = deployment_id.unwrap();
|
||||
if let Some(id) = deployment_id {
|
||||
newfm.id = id;
|
||||
}
|
||||
|
||||
fms[idx] = Some(newfm);
|
||||
|
||||
@@ -496,39 +496,32 @@ impl FileMeta {
|
||||
}
|
||||
|
||||
pub fn add_version_filemata(&mut self, ver: FileMetaVersion) -> Result<()> {
|
||||
let mod_time = ver.get_mod_time().unwrap().nanosecond();
|
||||
if !ver.valid() {
|
||||
return Err(Error::other("attempted to add invalid version"));
|
||||
}
|
||||
let encoded = ver.marshal_msg()?;
|
||||
|
||||
if self.versions.len() + 1 > 100 {
|
||||
if self.versions.len() + 1 >= 100 {
|
||||
return Err(Error::other(
|
||||
"You've exceeded the limit on the number of versions you can create on this object",
|
||||
));
|
||||
}
|
||||
|
||||
self.versions.push(FileMetaShallowVersion {
|
||||
header: FileMetaVersionHeader {
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
let mod_time = ver.get_mod_time();
|
||||
let encoded = ver.marshal_msg()?;
|
||||
let new_version = FileMetaShallowVersion {
|
||||
header: ver.header(),
|
||||
meta: encoded,
|
||||
};
|
||||
|
||||
let len = self.versions.len();
|
||||
for (i, existing) in self.versions.iter().enumerate() {
|
||||
if existing.header.mod_time.unwrap().nanosecond() <= mod_time {
|
||||
let vers = self.versions[i..len - 1].to_vec();
|
||||
self.versions[i + 1..].clone_from_slice(vers.as_slice());
|
||||
self.versions[i] = FileMetaShallowVersion {
|
||||
header: ver.header(),
|
||||
meta: encoded,
|
||||
};
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(Error::other("addVersion: Internal error, unable to add version"))
|
||||
// Find the insertion position: insert before the first element with mod_time >= new mod_time
|
||||
// This maintains descending order by mod_time (newest first)
|
||||
let insert_pos = self
|
||||
.versions
|
||||
.iter()
|
||||
.position(|existing| existing.header.mod_time <= mod_time)
|
||||
.unwrap_or(self.versions.len());
|
||||
self.versions.insert(insert_pos, new_version);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// delete_version deletes version, returns data_dir
|
||||
@@ -554,7 +547,15 @@ impl FileMeta {
|
||||
|
||||
match ver.header.version_type {
|
||||
VersionType::Invalid | VersionType::Legacy => return Err(Error::other("invalid file meta version")),
|
||||
VersionType::Delete => return Ok(None),
|
||||
VersionType::Delete => {
|
||||
self.versions.remove(i);
|
||||
if fi.deleted && fi.version_id.is_none() {
|
||||
self.add_version_filemata(ventry)?;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
return Ok(None);
|
||||
}
|
||||
VersionType::Object => {
|
||||
let v = self.get_idx(i)?;
|
||||
|
||||
@@ -600,6 +601,7 @@ impl FileMeta {
|
||||
|
||||
if fi.deleted {
|
||||
self.add_version_filemata(ventry)?;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
Err(Error::FileVersionNotFound)
|
||||
@@ -961,7 +963,8 @@ impl FileMetaVersion {
|
||||
|
||||
pub fn get_version_id(&self) -> Option<Uuid> {
|
||||
match self.version_type {
|
||||
VersionType::Object | VersionType::Delete => self.object.as_ref().map(|v| v.version_id).unwrap_or_default(),
|
||||
VersionType::Object => self.object.as_ref().map(|v| v.version_id).unwrap_or_default(),
|
||||
VersionType::Delete => self.delete_marker.as_ref().map(|v| v.version_id).unwrap_or_default(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -2363,7 +2366,7 @@ mod test {
|
||||
assert!(stats.delete_markers > 0, "应该有删除标记");
|
||||
|
||||
// 测试版本合并功能
|
||||
let merged = merge_file_meta_versions(1, false, 0, &[fm.versions.clone()]);
|
||||
let merged = merge_file_meta_versions(1, false, 0, std::slice::from_ref(&fm.versions));
|
||||
assert!(!merged.is_empty(), "合并后应该有版本");
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ pub mod fileinfo;
|
||||
mod filemeta;
|
||||
mod filemeta_inline;
|
||||
pub mod headers;
|
||||
mod metacache;
|
||||
pub mod metacache;
|
||||
|
||||
pub mod test_data;
|
||||
|
||||
|
||||
@@ -795,24 +795,26 @@ impl<T: Clone + Debug + Send + 'static> Cache<T> {
|
||||
}
|
||||
}
|
||||
|
||||
if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() * 2 {
|
||||
if self.updating.try_lock().is_ok() {
|
||||
let this = Arc::clone(&self);
|
||||
spawn(async move {
|
||||
let _ = this.update().await;
|
||||
});
|
||||
if self.opts.no_wait && now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() * 2 {
|
||||
if let Some(value) = v {
|
||||
if self.updating.try_lock().is_ok() {
|
||||
let this = Arc::clone(&self);
|
||||
spawn(async move {
|
||||
let _ = this.update().await;
|
||||
});
|
||||
}
|
||||
return Ok(value);
|
||||
}
|
||||
|
||||
return Ok(v.unwrap());
|
||||
}
|
||||
|
||||
let _ = self.updating.lock().await;
|
||||
|
||||
if let Ok(duration) =
|
||||
SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(AtomicOrdering::SeqCst)))
|
||||
{
|
||||
if let (Ok(duration), Some(value)) = (
|
||||
SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(AtomicOrdering::SeqCst))),
|
||||
v,
|
||||
) {
|
||||
if duration < self.ttl {
|
||||
return Ok(v.unwrap());
|
||||
return Ok(value);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -172,7 +172,12 @@ impl ObjectStore {
|
||||
}
|
||||
|
||||
if let Some(info) = v.item {
|
||||
let name = info.name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR);
|
||||
let object_name = if cfg!(target_os = "windows") {
|
||||
info.name.replace('\\', "/")
|
||||
} else {
|
||||
info.name
|
||||
};
|
||||
let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR);
|
||||
let _ = sender
|
||||
.send(StringOrErr {
|
||||
item: Some(name.to_owned()),
|
||||
|
||||
@@ -32,9 +32,7 @@ workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes.workspace = true
|
||||
futures.workspace = true
|
||||
lazy_static.workspace = true
|
||||
rustfs-protos.workspace = true
|
||||
rand.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
@@ -44,4 +42,3 @@ url.workspace = true
|
||||
uuid.workspace = true
|
||||
thiserror.workspace = true
|
||||
once_cell.workspace = true
|
||||
lru.workspace = true
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use once_cell::sync::Lazy;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::{client::LockClient, types::LockId};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct UnlockJob {
|
||||
lock_id: LockId,
|
||||
clients: Vec<Arc<dyn LockClient>>, // cloned Arcs; cheap and shares state
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct UnlockRuntime {
|
||||
tx: mpsc::Sender<UnlockJob>,
|
||||
}
|
||||
|
||||
// Global unlock runtime with background worker
|
||||
static UNLOCK_RUNTIME: Lazy<UnlockRuntime> = Lazy::new(|| {
|
||||
// Larger buffer to reduce contention during bursts
|
||||
let (tx, mut rx) = mpsc::channel::<UnlockJob>(8192);
|
||||
|
||||
// Spawn background worker when first used; assumes a Tokio runtime is available
|
||||
tokio::spawn(async move {
|
||||
while let Some(job) = rx.recv().await {
|
||||
// Best-effort release across clients; try all, success if any succeeds
|
||||
let mut any_ok = false;
|
||||
let lock_id = job.lock_id.clone();
|
||||
for client in job.clients.into_iter() {
|
||||
if client.release(&lock_id).await.unwrap_or(false) {
|
||||
any_ok = true;
|
||||
}
|
||||
}
|
||||
if !any_ok {
|
||||
tracing::warn!("LockGuard background release failed for {}", lock_id);
|
||||
} else {
|
||||
tracing::debug!("LockGuard background released {}", lock_id);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
UnlockRuntime { tx }
|
||||
});
|
||||
|
||||
/// A RAII guard that releases the lock asynchronously when dropped.
|
||||
#[derive(Debug)]
|
||||
pub struct LockGuard {
|
||||
lock_id: LockId,
|
||||
clients: Vec<Arc<dyn LockClient>>,
|
||||
/// If true, Drop will not try to release (used if user manually released).
|
||||
disarmed: bool,
|
||||
}
|
||||
|
||||
impl LockGuard {
|
||||
pub(crate) fn new(lock_id: LockId, clients: Vec<Arc<dyn LockClient>>) -> Self {
|
||||
Self {
|
||||
lock_id,
|
||||
clients,
|
||||
disarmed: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the lock id associated with this guard
|
||||
pub fn lock_id(&self) -> &LockId {
|
||||
&self.lock_id
|
||||
}
|
||||
|
||||
/// Manually disarm the guard so dropping it won't release the lock.
|
||||
/// Call this if you explicitly released the lock elsewhere.
|
||||
pub fn disarm(&mut self) {
|
||||
self.disarmed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for LockGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.disarmed {
|
||||
return;
|
||||
}
|
||||
|
||||
let job = UnlockJob {
|
||||
lock_id: self.lock_id.clone(),
|
||||
clients: self.clients.clone(),
|
||||
};
|
||||
|
||||
// Try a non-blocking send to avoid panics in Drop
|
||||
if let Err(err) = UNLOCK_RUNTIME.tx.try_send(job) {
|
||||
// Channel full or closed; best-effort fallback: spawn a detached task
|
||||
let lock_id = self.lock_id.clone();
|
||||
let clients = self.clients.clone();
|
||||
tracing::warn!("LockGuard channel send failed ({}), spawning fallback unlock task for {}", err, lock_id);
|
||||
|
||||
// If runtime is not available, this will panic; but in RustFS we are inside Tokio contexts.
|
||||
let handle = tokio::spawn(async move {
|
||||
let futures_iter = clients.into_iter().map(|client| {
|
||||
let id = lock_id.clone();
|
||||
async move { client.release(&id).await.unwrap_or(false) }
|
||||
});
|
||||
let _ = futures::future::join_all(futures_iter).await;
|
||||
});
|
||||
// Explicitly drop the JoinHandle to acknowledge detaching the task.
|
||||
std::mem::drop(handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,7 @@ pub mod local;
|
||||
|
||||
// Core Modules
|
||||
pub mod error;
|
||||
pub mod guard;
|
||||
pub mod types;
|
||||
|
||||
// ============================================================================
|
||||
@@ -39,6 +40,7 @@ pub use crate::{
|
||||
client::{LockClient, local::LocalClient, remote::RemoteClient},
|
||||
// Error types
|
||||
error::{LockError, Result},
|
||||
guard::LockGuard,
|
||||
local::LocalLockMap,
|
||||
// Main components
|
||||
namespace::{NamespaceLock, NamespaceLockManager},
|
||||
|
||||
+331
-157
@@ -12,11 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::sync::{Mutex, Notify, RwLock};
|
||||
|
||||
use crate::LockRequest;
|
||||
|
||||
@@ -29,6 +29,11 @@ pub struct LocalLockEntry {
|
||||
pub readers: HashMap<String, usize>,
|
||||
/// lock expiration time
|
||||
pub expires_at: Option<Instant>,
|
||||
/// number of writers waiting (for simple fairness against reader storms)
|
||||
pub writer_pending: usize,
|
||||
/// notifiers for readers/writers
|
||||
pub notify_readers: Arc<Notify>,
|
||||
pub notify_writers: Arc<Notify>,
|
||||
}
|
||||
|
||||
/// local lock map
|
||||
@@ -38,6 +43,10 @@ pub struct LocalLockMap {
|
||||
pub locks: Arc<RwLock<HashMap<crate::types::LockId, Arc<RwLock<LocalLockEntry>>>>>,
|
||||
/// Shutdown flag for background tasks
|
||||
shutdown: Arc<AtomicBool>,
|
||||
/// expiration schedule map: when -> lock_ids
|
||||
expirations: Arc<Mutex<BTreeMap<Instant, Vec<crate::types::LockId>>>>,
|
||||
/// notify expiry task when new earlier deadline arrives
|
||||
exp_notify: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl Default for LocalLockMap {
|
||||
@@ -52,6 +61,8 @@ impl LocalLockMap {
|
||||
let map = Self {
|
||||
locks: Arc::new(RwLock::new(HashMap::new())),
|
||||
shutdown: Arc::new(AtomicBool::new(false)),
|
||||
expirations: Arc::new(Mutex::new(BTreeMap::new())),
|
||||
exp_notify: Arc::new(Notify::new()),
|
||||
};
|
||||
map.spawn_expiry_task();
|
||||
map
|
||||
@@ -61,56 +72,115 @@ impl LocalLockMap {
|
||||
fn spawn_expiry_task(&self) {
|
||||
let locks = self.locks.clone();
|
||||
let shutdown = self.shutdown.clone();
|
||||
let expirations = self.expirations.clone();
|
||||
let exp_notify = self.exp_notify.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(1));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
if shutdown.load(Ordering::Relaxed) {
|
||||
tracing::debug!("Expiry task shutting down");
|
||||
break;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
let mut to_remove = Vec::new();
|
||||
// Find next deadline and drain due ids
|
||||
let (due_ids, wait_duration) = {
|
||||
let mut due = Vec::new();
|
||||
let mut guard = expirations.lock().await;
|
||||
let now = Instant::now();
|
||||
let next_deadline = guard.first_key_value().map(|(k, _)| *k);
|
||||
// drain all <= now
|
||||
let mut keys_to_remove = Vec::new();
|
||||
for (k, v) in guard.range(..=now).map(|(k, v)| (*k, v.clone())) {
|
||||
due.extend(v);
|
||||
keys_to_remove.push(k);
|
||||
}
|
||||
for k in keys_to_remove {
|
||||
guard.remove(&k);
|
||||
}
|
||||
let wait = if due.is_empty() {
|
||||
next_deadline.map(|dl| if dl > now { dl - now } else { Duration::from_millis(0) })
|
||||
} else {
|
||||
Some(Duration::from_millis(0))
|
||||
};
|
||||
(due, wait)
|
||||
};
|
||||
|
||||
{
|
||||
let locks_guard = locks.read().await;
|
||||
for (key, entry) in locks_guard.iter() {
|
||||
if let Ok(mut entry_guard) = entry.try_write() {
|
||||
if let Some(exp) = entry_guard.expires_at {
|
||||
if exp <= now {
|
||||
entry_guard.writer = None;
|
||||
entry_guard.readers.clear();
|
||||
entry_guard.expires_at = None;
|
||||
if !due_ids.is_empty() {
|
||||
// process due ids without holding the map lock during awaits
|
||||
let now = Instant::now();
|
||||
// collect entries to process
|
||||
let entries: Vec<(crate::types::LockId, Arc<RwLock<LocalLockEntry>>)> = {
|
||||
let locks_guard = locks.read().await;
|
||||
due_ids
|
||||
.into_iter()
|
||||
.filter_map(|id| locks_guard.get(&id).cloned().map(|e| (id, e)))
|
||||
.collect()
|
||||
};
|
||||
|
||||
if entry_guard.writer.is_none() && entry_guard.readers.is_empty() {
|
||||
to_remove.push(key.clone());
|
||||
}
|
||||
let mut to_remove = Vec::new();
|
||||
for (lock_id, entry) in entries {
|
||||
let mut entry_guard = entry.write().await;
|
||||
if let Some(exp) = entry_guard.expires_at {
|
||||
if exp <= now {
|
||||
entry_guard.writer = None;
|
||||
entry_guard.readers.clear();
|
||||
entry_guard.expires_at = None;
|
||||
entry_guard.notify_writers.notify_waiters();
|
||||
entry_guard.notify_readers.notify_waiters();
|
||||
if entry_guard.writer.is_none() && entry_guard.readers.is_empty() {
|
||||
to_remove.push(lock_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !to_remove.is_empty() {
|
||||
let mut locks_w = locks.write().await;
|
||||
for id in to_remove {
|
||||
let _ = locks_w.remove(&id);
|
||||
}
|
||||
}
|
||||
continue; // immediately look for next
|
||||
}
|
||||
|
||||
if !to_remove.is_empty() {
|
||||
let mut locks_guard = locks.write().await;
|
||||
for key in to_remove {
|
||||
locks_guard.remove(&key);
|
||||
// nothing due; wait for next deadline or notification
|
||||
if let Some(dur) = wait_duration {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(dur) => {},
|
||||
_ = exp_notify.notified() => {},
|
||||
}
|
||||
} else {
|
||||
// no deadlines, wait for new schedule or shutdown tick
|
||||
exp_notify.notified().await;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// schedule an expiry time for the given lock id (inline, avoid per-acquisition spawn)
|
||||
async fn schedule_expiry(&self, id: crate::types::LockId, exp: Instant) {
|
||||
let mut guard = self.expirations.lock().await;
|
||||
let is_earliest = match guard.first_key_value() {
|
||||
Some((k, _)) => exp < *k,
|
||||
None => true,
|
||||
};
|
||||
guard.entry(exp).or_insert_with(Vec::new).push(id);
|
||||
drop(guard);
|
||||
if is_earliest {
|
||||
self.exp_notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
/// write lock with TTL, support timeout, use LockRequest
|
||||
pub async fn lock_with_ttl_id(&self, request: &LockRequest) -> std::io::Result<bool> {
|
||||
let start = Instant::now();
|
||||
let expires_at = Some(Instant::now() + request.ttl);
|
||||
|
||||
loop {
|
||||
// get or create lock entry
|
||||
let entry = {
|
||||
// get or create lock entry (double-checked to reduce write-lock contention)
|
||||
let entry = if let Some(e) = {
|
||||
let locks_guard = self.locks.read().await;
|
||||
locks_guard.get(&request.lock_id).cloned()
|
||||
} {
|
||||
e
|
||||
} else {
|
||||
let mut locks_guard = self.locks.write().await;
|
||||
locks_guard
|
||||
.entry(request.lock_id.clone())
|
||||
@@ -119,13 +189,17 @@ impl LocalLockMap {
|
||||
writer: None,
|
||||
readers: HashMap::new(),
|
||||
expires_at: None,
|
||||
writer_pending: 0,
|
||||
notify_readers: Arc::new(Notify::new()),
|
||||
notify_writers: Arc::new(Notify::new()),
|
||||
}))
|
||||
})
|
||||
.clone()
|
||||
};
|
||||
|
||||
// try to get write lock to modify state
|
||||
if let Ok(mut entry_guard) = entry.try_write() {
|
||||
// attempt acquisition or wait using Notify
|
||||
let notify_to_wait = {
|
||||
let mut entry_guard = entry.write().await;
|
||||
// check expired state
|
||||
let now = Instant::now();
|
||||
if let Some(exp) = entry_guard.expires_at {
|
||||
@@ -136,30 +210,68 @@ impl LocalLockMap {
|
||||
}
|
||||
}
|
||||
|
||||
// check if can get write lock
|
||||
// try acquire
|
||||
if entry_guard.writer.is_none() && entry_guard.readers.is_empty() {
|
||||
entry_guard.writer = Some(request.owner.clone());
|
||||
entry_guard.expires_at = expires_at;
|
||||
let expires_at = Instant::now() + request.ttl;
|
||||
entry_guard.expires_at = Some(expires_at);
|
||||
tracing::debug!("Write lock acquired for resource '{}' by owner '{}'", request.resource, request.owner);
|
||||
{
|
||||
drop(entry_guard);
|
||||
self.schedule_expiry(request.lock_id.clone(), expires_at).await;
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
// couldn't acquire now, mark as pending writer and choose notifier
|
||||
entry_guard.writer_pending = entry_guard.writer_pending.saturating_add(1);
|
||||
entry_guard.notify_writers.clone()
|
||||
};
|
||||
|
||||
if start.elapsed() >= request.acquire_timeout {
|
||||
// wait with remaining timeout
|
||||
let elapsed = start.elapsed();
|
||||
if elapsed >= request.acquire_timeout {
|
||||
// best-effort decrement pending counter
|
||||
if let Ok(mut eg) = entry.try_write() {
|
||||
eg.writer_pending = eg.writer_pending.saturating_sub(1);
|
||||
} else {
|
||||
let mut eg = entry.write().await;
|
||||
eg.writer_pending = eg.writer_pending.saturating_sub(1);
|
||||
}
|
||||
return Ok(false);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
let remaining = request.acquire_timeout - elapsed;
|
||||
if tokio::time::timeout(remaining, notify_to_wait.notified()).await.is_err() {
|
||||
// timeout; decrement pending before returning
|
||||
if let Ok(mut eg) = entry.try_write() {
|
||||
eg.writer_pending = eg.writer_pending.saturating_sub(1);
|
||||
} else {
|
||||
let mut eg = entry.write().await;
|
||||
eg.writer_pending = eg.writer_pending.saturating_sub(1);
|
||||
}
|
||||
return Ok(false);
|
||||
}
|
||||
// woke up; decrement pending before retrying
|
||||
if let Ok(mut eg) = entry.try_write() {
|
||||
eg.writer_pending = eg.writer_pending.saturating_sub(1);
|
||||
} else {
|
||||
let mut eg = entry.write().await;
|
||||
eg.writer_pending = eg.writer_pending.saturating_sub(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// read lock with TTL, support timeout, use LockRequest
|
||||
pub async fn rlock_with_ttl_id(&self, request: &LockRequest) -> std::io::Result<bool> {
|
||||
let start = Instant::now();
|
||||
let expires_at = Some(Instant::now() + request.ttl);
|
||||
|
||||
loop {
|
||||
// get or create lock entry
|
||||
let entry = {
|
||||
// get or create lock entry (double-checked to reduce write-lock contention)
|
||||
let entry = if let Some(e) = {
|
||||
let locks_guard = self.locks.read().await;
|
||||
locks_guard.get(&request.lock_id).cloned()
|
||||
} {
|
||||
e
|
||||
} else {
|
||||
let mut locks_guard = self.locks.write().await;
|
||||
locks_guard
|
||||
.entry(request.lock_id.clone())
|
||||
@@ -168,13 +280,17 @@ impl LocalLockMap {
|
||||
writer: None,
|
||||
readers: HashMap::new(),
|
||||
expires_at: None,
|
||||
writer_pending: 0,
|
||||
notify_readers: Arc::new(Notify::new()),
|
||||
notify_writers: Arc::new(Notify::new()),
|
||||
}))
|
||||
})
|
||||
.clone()
|
||||
};
|
||||
|
||||
// try to get write lock to modify state
|
||||
if let Ok(mut entry_guard) = entry.try_write() {
|
||||
// attempt acquisition or wait using Notify
|
||||
let notify_to_wait = {
|
||||
let mut entry_guard = entry.write().await;
|
||||
// check expired state
|
||||
let now = Instant::now();
|
||||
if let Some(exp) = entry_guard.expires_at {
|
||||
@@ -185,189 +301,247 @@ impl LocalLockMap {
|
||||
}
|
||||
}
|
||||
|
||||
// check if can get read lock
|
||||
if entry_guard.writer.is_none() {
|
||||
// increase read lock count
|
||||
if entry_guard.writer.is_none() && entry_guard.writer_pending == 0 {
|
||||
*entry_guard.readers.entry(request.owner.clone()).or_insert(0) += 1;
|
||||
entry_guard.expires_at = expires_at;
|
||||
let expires_at = Instant::now() + request.ttl;
|
||||
entry_guard.expires_at = Some(expires_at);
|
||||
tracing::debug!("Read lock acquired for resource '{}' by owner '{}'", request.resource, request.owner);
|
||||
{
|
||||
drop(entry_guard);
|
||||
self.schedule_expiry(request.lock_id.clone(), expires_at).await;
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
|
||||
if start.elapsed() >= request.acquire_timeout {
|
||||
// choose notifier: prefer waiting on writers if writers pending, else readers
|
||||
if entry_guard.writer_pending > 0 {
|
||||
entry_guard.notify_writers.clone()
|
||||
} else {
|
||||
entry_guard.notify_readers.clone()
|
||||
}
|
||||
};
|
||||
|
||||
// wait with remaining timeout
|
||||
let elapsed = start.elapsed();
|
||||
if elapsed >= request.acquire_timeout {
|
||||
return Ok(false);
|
||||
}
|
||||
let remaining = request.acquire_timeout - elapsed;
|
||||
if tokio::time::timeout(remaining, notify_to_wait.notified()).await.is_err() {
|
||||
return Ok(false);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// unlock by LockId and owner - need to specify owner to correctly unlock
|
||||
pub async fn unlock_by_id_and_owner(&self, lock_id: &crate::types::LockId, owner: &str) -> std::io::Result<()> {
|
||||
println!("Unlocking lock_id: {lock_id:?}, owner: {owner}");
|
||||
let mut need_remove = false;
|
||||
|
||||
{
|
||||
// first, get the entry without holding the write lock on the map
|
||||
let entry = {
|
||||
let locks_guard = self.locks.read().await;
|
||||
if let Some(entry) = locks_guard.get(lock_id) {
|
||||
println!("Found lock entry, attempting to acquire write lock...");
|
||||
match entry.try_write() {
|
||||
Ok(mut entry_guard) => {
|
||||
println!("Successfully acquired write lock for unlock");
|
||||
// try to release write lock
|
||||
if entry_guard.writer.as_ref() == Some(&owner.to_string()) {
|
||||
println!("Releasing write lock for owner: {owner}");
|
||||
entry_guard.writer = None;
|
||||
}
|
||||
// try to release read lock
|
||||
else if let Some(count) = entry_guard.readers.get_mut(owner) {
|
||||
println!("Releasing read lock for owner: {owner} (count: {count})");
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
entry_guard.readers.remove(owner);
|
||||
println!("Removed owner {owner} from readers");
|
||||
}
|
||||
} else {
|
||||
println!("Owner {owner} not found in writers or readers");
|
||||
}
|
||||
// check if need to remove
|
||||
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
|
||||
println!("Lock entry is empty, marking for removal");
|
||||
entry_guard.expires_at = None;
|
||||
need_remove = true;
|
||||
} else {
|
||||
println!(
|
||||
"Lock entry still has content: writer={:?}, readers={:?}",
|
||||
entry_guard.writer, entry_guard.readers
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
println!("Failed to acquire write lock for unlock - this is the problem!");
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"Failed to acquire write lock for unlock",
|
||||
));
|
||||
}
|
||||
match locks_guard.get(lock_id) {
|
||||
Some(e) => e.clone(),
|
||||
None => return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "Lock entry not found")),
|
||||
}
|
||||
};
|
||||
|
||||
let mut need_remove = false;
|
||||
let (notify_writers, notify_readers, writer_pending, writer_none) = {
|
||||
let mut entry_guard = entry.write().await;
|
||||
|
||||
// try to release write lock
|
||||
if entry_guard.writer.as_ref() == Some(&owner.to_string()) {
|
||||
entry_guard.writer = None;
|
||||
}
|
||||
// try to release read lock
|
||||
else if let Some(count) = entry_guard.readers.get_mut(owner) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
entry_guard.readers.remove(owner);
|
||||
}
|
||||
} else {
|
||||
println!("Lock entry not found for lock_id: {lock_id:?}");
|
||||
// owner not found, treat as no-op
|
||||
}
|
||||
|
||||
// check if need to remove
|
||||
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
|
||||
entry_guard.expires_at = None;
|
||||
need_remove = true;
|
||||
}
|
||||
|
||||
// capture notifications and state
|
||||
(
|
||||
entry_guard.notify_writers.clone(),
|
||||
entry_guard.notify_readers.clone(),
|
||||
entry_guard.writer_pending,
|
||||
entry_guard.writer.is_none(),
|
||||
)
|
||||
};
|
||||
|
||||
if writer_pending > 0 && writer_none {
|
||||
// Wake a single writer to preserve fairness and avoid thundering herd
|
||||
notify_writers.notify_one();
|
||||
} else if writer_none {
|
||||
// No writers waiting, allow readers to proceed
|
||||
notify_readers.notify_waiters();
|
||||
}
|
||||
|
||||
// only here, entry's Ref is really dropped, can safely remove
|
||||
if need_remove {
|
||||
println!("Removing lock entry from map...");
|
||||
let mut locks_guard = self.locks.write().await;
|
||||
let removed = locks_guard.remove(lock_id);
|
||||
println!("Lock entry removed: {:?}", removed.is_some());
|
||||
let _ = locks_guard.remove(lock_id);
|
||||
}
|
||||
println!("Unlock operation completed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// unlock by LockId - smart release (compatible with old interface, but may be inaccurate)
|
||||
pub async fn unlock_by_id(&self, lock_id: &crate::types::LockId) -> std::io::Result<()> {
|
||||
let mut need_remove = false;
|
||||
|
||||
{
|
||||
let entry = {
|
||||
let locks_guard = self.locks.read().await;
|
||||
if let Some(entry) = locks_guard.get(lock_id) {
|
||||
if let Ok(mut entry_guard) = entry.try_write() {
|
||||
// release write lock first
|
||||
if entry_guard.writer.is_some() {
|
||||
entry_guard.writer = None;
|
||||
}
|
||||
// if no write lock, release first read lock
|
||||
else if let Some((owner, _)) = entry_guard.readers.iter().next() {
|
||||
let owner = owner.clone();
|
||||
if let Some(count) = entry_guard.readers.get_mut(&owner) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
entry_guard.readers.remove(&owner);
|
||||
}
|
||||
}
|
||||
}
|
||||
match locks_guard.get(lock_id) {
|
||||
Some(e) => e.clone(),
|
||||
None => return Ok(()), // nothing to do
|
||||
}
|
||||
};
|
||||
|
||||
// if completely idle, clean entry
|
||||
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
|
||||
entry_guard.expires_at = None;
|
||||
need_remove = true;
|
||||
let mut need_remove = false;
|
||||
let (notify_writers, notify_readers, writer_pending, writer_none) = {
|
||||
let mut entry_guard = entry.write().await;
|
||||
|
||||
// release write lock first
|
||||
if entry_guard.writer.is_some() {
|
||||
entry_guard.writer = None;
|
||||
}
|
||||
// if no write lock, release first read lock
|
||||
else if let Some((owner, _)) = entry_guard.readers.iter().next() {
|
||||
let owner = owner.clone();
|
||||
if let Some(count) = entry_guard.readers.get_mut(&owner) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
entry_guard.readers.remove(&owner);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
|
||||
entry_guard.expires_at = None;
|
||||
need_remove = true;
|
||||
}
|
||||
|
||||
(
|
||||
entry_guard.notify_writers.clone(),
|
||||
entry_guard.notify_readers.clone(),
|
||||
entry_guard.writer_pending,
|
||||
entry_guard.writer.is_none(),
|
||||
)
|
||||
};
|
||||
|
||||
if writer_pending > 0 && writer_none {
|
||||
notify_writers.notify_one();
|
||||
} else if writer_none {
|
||||
notify_readers.notify_waiters();
|
||||
}
|
||||
|
||||
if need_remove {
|
||||
let mut locks_guard = self.locks.write().await;
|
||||
locks_guard.remove(lock_id);
|
||||
let _ = locks_guard.remove(lock_id);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// runlock by LockId and owner - need to specify owner to correctly unlock read lock
|
||||
pub async fn runlock_by_id_and_owner(&self, lock_id: &crate::types::LockId, owner: &str) -> std::io::Result<()> {
|
||||
let mut need_remove = false;
|
||||
|
||||
{
|
||||
let entry = {
|
||||
let locks_guard = self.locks.read().await;
|
||||
if let Some(entry) = locks_guard.get(lock_id) {
|
||||
if let Ok(mut entry_guard) = entry.try_write() {
|
||||
// release read lock
|
||||
if let Some(count) = entry_guard.readers.get_mut(owner) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
entry_guard.readers.remove(owner);
|
||||
}
|
||||
}
|
||||
match locks_guard.get(lock_id) {
|
||||
Some(e) => e.clone(),
|
||||
None => return Ok(()),
|
||||
}
|
||||
};
|
||||
|
||||
// if completely idle, clean entry
|
||||
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
|
||||
entry_guard.expires_at = None;
|
||||
need_remove = true;
|
||||
}
|
||||
let mut need_remove = false;
|
||||
let (notify_writers, notify_readers, writer_pending, writer_none) = {
|
||||
let mut entry_guard = entry.write().await;
|
||||
|
||||
// release read lock
|
||||
if let Some(count) = entry_guard.readers.get_mut(owner) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
entry_guard.readers.remove(owner);
|
||||
}
|
||||
}
|
||||
|
||||
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
|
||||
entry_guard.expires_at = None;
|
||||
need_remove = true;
|
||||
}
|
||||
|
||||
(
|
||||
entry_guard.notify_writers.clone(),
|
||||
entry_guard.notify_readers.clone(),
|
||||
entry_guard.writer_pending,
|
||||
entry_guard.writer.is_none(),
|
||||
)
|
||||
};
|
||||
|
||||
if writer_pending > 0 && writer_none {
|
||||
notify_writers.notify_waiters();
|
||||
} else if writer_none {
|
||||
notify_readers.notify_waiters();
|
||||
}
|
||||
|
||||
if need_remove {
|
||||
let mut locks_guard = self.locks.write().await;
|
||||
locks_guard.remove(lock_id);
|
||||
let _ = locks_guard.remove(lock_id);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// runlock by LockId - smart release read lock (compatible with old interface)
|
||||
pub async fn runlock_by_id(&self, lock_id: &crate::types::LockId) -> std::io::Result<()> {
|
||||
let mut need_remove = false;
|
||||
|
||||
{
|
||||
let entry = {
|
||||
let locks_guard = self.locks.read().await;
|
||||
if let Some(entry) = locks_guard.get(lock_id) {
|
||||
if let Ok(mut entry_guard) = entry.try_write() {
|
||||
// release first read lock
|
||||
if let Some((owner, _)) = entry_guard.readers.iter().next() {
|
||||
let owner = owner.clone();
|
||||
if let Some(count) = entry_guard.readers.get_mut(&owner) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
entry_guard.readers.remove(&owner);
|
||||
}
|
||||
}
|
||||
}
|
||||
match locks_guard.get(lock_id) {
|
||||
Some(e) => e.clone(),
|
||||
None => return Ok(()),
|
||||
}
|
||||
};
|
||||
|
||||
// if completely idle, clean entry
|
||||
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
|
||||
entry_guard.expires_at = None;
|
||||
need_remove = true;
|
||||
let mut need_remove = false;
|
||||
let (notify_writers, notify_readers, writer_pending, writer_none) = {
|
||||
let mut entry_guard = entry.write().await;
|
||||
|
||||
// release first read lock
|
||||
if let Some((owner, _)) = entry_guard.readers.iter().next() {
|
||||
let owner = owner.clone();
|
||||
if let Some(count) = entry_guard.readers.get_mut(&owner) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
entry_guard.readers.remove(&owner);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
|
||||
entry_guard.expires_at = None;
|
||||
need_remove = true;
|
||||
}
|
||||
|
||||
(
|
||||
entry_guard.notify_writers.clone(),
|
||||
entry_guard.notify_readers.clone(),
|
||||
entry_guard.writer_pending,
|
||||
entry_guard.writer.is_none(),
|
||||
)
|
||||
};
|
||||
|
||||
if writer_pending > 0 && writer_none {
|
||||
notify_writers.notify_waiters();
|
||||
} else if writer_none {
|
||||
notify_readers.notify_waiters();
|
||||
}
|
||||
|
||||
if need_remove {
|
||||
let mut locks_guard = self.locks.write().await;
|
||||
locks_guard.remove(lock_id);
|
||||
let _ = locks_guard.remove(lock_id);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+109
-39
@@ -19,6 +19,7 @@ use std::time::Duration;
|
||||
use crate::{
|
||||
client::LockClient,
|
||||
error::{LockError, Result},
|
||||
guard::LockGuard,
|
||||
types::{LockId, LockInfo, LockRequest, LockResponse, LockStatus, LockType},
|
||||
};
|
||||
|
||||
@@ -60,6 +61,22 @@ impl NamespaceLock {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create namespace lock with clients and an explicit quorum size.
|
||||
/// Quorum will be clamped into [1, clients.len()]. For single client, quorum is always 1.
|
||||
pub fn with_clients_and_quorum(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
|
||||
let q = if clients.len() <= 1 {
|
||||
1
|
||||
} else {
|
||||
quorum.clamp(1, clients.len())
|
||||
};
|
||||
|
||||
Self {
|
||||
clients,
|
||||
namespace,
|
||||
quorum: q,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create namespace lock with client (compatibility)
|
||||
pub fn with_client(client: Arc<dyn LockClient>) -> Self {
|
||||
Self::with_clients("default".to_string(), vec![client])
|
||||
@@ -86,54 +103,77 @@ impl NamespaceLock {
|
||||
return self.clients[0].acquire_lock(request).await;
|
||||
}
|
||||
|
||||
// Two-phase commit for distributed lock acquisition
|
||||
self.acquire_lock_with_2pc(request).await
|
||||
// Quorum-based acquisition for distributed mode
|
||||
let (resp, _idxs) = self.acquire_lock_quorum(request).await?;
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
/// Two-phase commit lock acquisition: all nodes must succeed or all fail
|
||||
async fn acquire_lock_with_2pc(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
// Phase 1: Prepare - try to acquire lock on all clients
|
||||
let futures: Vec<_> = self
|
||||
/// Acquire a lock and return a RAII guard that will release asynchronously on Drop.
|
||||
/// This is a thin wrapper around `acquire_lock` and will only create a guard when acquisition succeeds.
|
||||
pub async fn acquire_guard(&self, request: &LockRequest) -> Result<Option<LockGuard>> {
|
||||
if self.clients.is_empty() {
|
||||
return Err(LockError::internal("No lock clients available"));
|
||||
}
|
||||
|
||||
if self.clients.len() == 1 {
|
||||
let resp = self.clients[0].acquire_lock(request).await?;
|
||||
if resp.success {
|
||||
return Ok(Some(LockGuard::new(
|
||||
LockId::new_deterministic(&request.resource),
|
||||
vec![self.clients[0].clone()],
|
||||
)));
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let (resp, idxs) = self.acquire_lock_quorum(request).await?;
|
||||
if resp.success {
|
||||
let subset: Vec<_> = idxs.into_iter().filter_map(|i| self.clients.get(i).cloned()).collect();
|
||||
Ok(Some(LockGuard::new(LockId::new_deterministic(&request.resource), subset)))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience: acquire exclusive lock as a guard
|
||||
pub async fn lock_guard(&self, resource: &str, owner: &str, timeout: Duration, ttl: Duration) -> Result<Option<LockGuard>> {
|
||||
let req = LockRequest::new(self.get_resource_key(resource), LockType::Exclusive, owner)
|
||||
.with_acquire_timeout(timeout)
|
||||
.with_ttl(ttl);
|
||||
self.acquire_guard(&req).await
|
||||
}
|
||||
|
||||
/// Convenience: acquire shared lock as a guard
|
||||
pub async fn rlock_guard(&self, resource: &str, owner: &str, timeout: Duration, ttl: Duration) -> Result<Option<LockGuard>> {
|
||||
let req = LockRequest::new(self.get_resource_key(resource), LockType::Shared, owner)
|
||||
.with_acquire_timeout(timeout)
|
||||
.with_ttl(ttl);
|
||||
self.acquire_guard(&req).await
|
||||
}
|
||||
|
||||
/// Quorum-based lock acquisition: success if at least `self.quorum` clients succeed.
|
||||
/// Returns the LockResponse and the indices of clients that acquired the lock.
|
||||
async fn acquire_lock_quorum(&self, request: &LockRequest) -> Result<(LockResponse, Vec<usize>)> {
|
||||
let futs: Vec<_> = self
|
||||
.clients
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, client)| async move {
|
||||
let result = client.acquire_lock(request).await;
|
||||
(idx, result)
|
||||
})
|
||||
.map(|(idx, client)| async move { (idx, client.acquire_lock(request).await) })
|
||||
.collect();
|
||||
|
||||
let results = futures::future::join_all(futures).await;
|
||||
let results = futures::future::join_all(futs).await;
|
||||
let mut successful_clients = Vec::new();
|
||||
let mut failed_clients = Vec::new();
|
||||
|
||||
// Collect results
|
||||
for (idx, result) in results {
|
||||
match result {
|
||||
Ok(response) if response.success => {
|
||||
for (idx, res) in results {
|
||||
if let Ok(resp) = res {
|
||||
if resp.success {
|
||||
successful_clients.push(idx);
|
||||
}
|
||||
_ => {
|
||||
failed_clients.push(idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we have enough successful acquisitions for quorum
|
||||
if successful_clients.len() >= self.quorum {
|
||||
// Phase 2a: Commit - we have quorum, but need to ensure consistency
|
||||
// If not all clients succeeded, we need to rollback for consistency
|
||||
if successful_clients.len() < self.clients.len() {
|
||||
// Rollback all successful acquisitions to maintain consistency
|
||||
self.rollback_acquisitions(request, &successful_clients).await;
|
||||
return Ok(LockResponse::failure(
|
||||
"Partial success detected, rolled back for consistency".to_string(),
|
||||
Duration::ZERO,
|
||||
));
|
||||
}
|
||||
|
||||
// All clients succeeded - lock acquired successfully
|
||||
Ok(LockResponse::success(
|
||||
let resp = LockResponse::success(
|
||||
LockInfo {
|
||||
id: LockId::new_deterministic(&request.resource),
|
||||
resource: request.resource.clone(),
|
||||
@@ -148,16 +188,17 @@ impl NamespaceLock {
|
||||
wait_start_time: None,
|
||||
},
|
||||
Duration::ZERO,
|
||||
))
|
||||
);
|
||||
Ok((resp, successful_clients))
|
||||
} else {
|
||||
// Phase 2b: Abort - insufficient quorum, rollback any successful acquisitions
|
||||
if !successful_clients.is_empty() {
|
||||
self.rollback_acquisitions(request, &successful_clients).await;
|
||||
}
|
||||
Ok(LockResponse::failure(
|
||||
let resp = LockResponse::failure(
|
||||
format!("Failed to acquire quorum: {}/{} required", successful_clients.len(), self.quorum),
|
||||
Duration::ZERO,
|
||||
))
|
||||
);
|
||||
Ok((resp, Vec::new()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -420,6 +461,33 @@ mod tests {
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_guard_acquire_and_drop_release() {
|
||||
let ns_lock = NamespaceLock::with_client(Arc::new(LocalClient::new()));
|
||||
|
||||
// Acquire guard
|
||||
let guard = ns_lock
|
||||
.lock_guard("guard-resource", "owner", Duration::from_millis(100), Duration::from_secs(5))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(guard.is_some());
|
||||
let lock_id = guard.as_ref().unwrap().lock_id().clone();
|
||||
|
||||
// Drop guard to trigger background release
|
||||
drop(guard);
|
||||
|
||||
// Give background worker a moment to process
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
|
||||
// Re-acquire should succeed (previous lock released)
|
||||
let req = LockRequest::new(&lock_id.resource, LockType::Exclusive, "owner").with_ttl(Duration::from_secs(2));
|
||||
let resp = ns_lock.acquire_lock(&req).await.unwrap();
|
||||
assert!(resp.success);
|
||||
|
||||
// Cleanup
|
||||
let _ = ns_lock.release_lock(&LockId::new_deterministic(&lock_id.resource)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_connection_health() {
|
||||
let local_lock = NamespaceLock::new("test-namespace".to_string());
|
||||
@@ -502,9 +570,11 @@ mod tests {
|
||||
let client2: Arc<dyn LockClient> = Arc::new(LocalClient::new());
|
||||
let clients = vec![client1, client2];
|
||||
|
||||
let ns_lock = NamespaceLock::with_clients("test-namespace".to_string(), clients);
|
||||
// LocalClient shares a global in-memory map. For exclusive locks, only one can acquire at a time.
|
||||
// In real distributed setups the quorum should be tied to EC write quorum. Here we use quorum=1 for success.
|
||||
let ns_lock = NamespaceLock::with_clients_and_quorum("test-namespace".to_string(), clients, 1);
|
||||
|
||||
let request = LockRequest::new("test-resource", LockType::Exclusive, "test_owner").with_ttl(Duration::from_secs(10));
|
||||
let request = LockRequest::new("test-resource", LockType::Shared, "test_owner").with_ttl(Duration::from_secs(2));
|
||||
|
||||
// This should succeed only if ALL clients can acquire the lock
|
||||
let response = ns_lock.acquire_lock(&request).await.unwrap();
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
FROM rust:1.88 AS builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN cargo build --release -p rustfs-mcp
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /build/target/release/rustfs-mcp /app/
|
||||
|
||||
ENTRYPOINT ["/app/rustfs-mcp"]
|
||||
@@ -98,7 +98,9 @@ rustfs-mcp --log-level debug --region us-west-2
|
||||
```
|
||||
|
||||
### Integration with chat client
|
||||
|
||||
#### Option 1: Using Command Line Arguments
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
@@ -116,6 +118,7 @@ rustfs-mcp --log-level debug --region us-west-2
|
||||
```
|
||||
|
||||
#### Option 2: Using Environment Variables
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
@@ -130,26 +133,84 @@ rustfs-mcp --log-level debug --region us-west-2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Using MCP with Docker
|
||||
|
||||
#### Docker image build
|
||||
|
||||
Using MCP with docker will simply the usage of rustfs mcp. Building the docker image with below command:
|
||||
|
||||
```
|
||||
docker build -f Dockerfile -t rustfs/rustfs-mcp ../../
|
||||
```
|
||||
|
||||
Alternatively, if you want to build the image from the rustfs codebase root directory,run the command:
|
||||
|
||||
```
|
||||
docker build -f crates/mcp/Dockerfile -t rustfs/rustfs-mcp .
|
||||
```
|
||||
|
||||
#### IDE Configuration
|
||||
|
||||
Adding the following content in IDE MCP settings:
|
||||
|
||||
```
|
||||
{
|
||||
"mcpServers": {
|
||||
"rustfs-mcp": {
|
||||
"command": "docker",
|
||||
"args": [
|
||||
"run",
|
||||
"--rm",
|
||||
"-i",
|
||||
"-e",
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"-e",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"-e",
|
||||
"AWS_REGION",
|
||||
"-e",
|
||||
"AWS_ENDPOINT_URL",
|
||||
"rustfs/rustfs-mcp"
|
||||
],
|
||||
"env": {
|
||||
"AWS_ACCESS_KEY_ID": "rustfs_access_key",
|
||||
"AWS_SECRET_ACCESS_KEY": "rustfs_secret_key",
|
||||
"AWS_REGION": "cn-east-1",
|
||||
"AWS_ENDPOINT_URL": "rustfs_instance_url"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If success, MCP configure page will show the [available tools](#️-available-tools).
|
||||
|
||||
## 🛠️ Available Tools
|
||||
|
||||
The MCP server exposes the following tools that AI assistants can use:
|
||||
|
||||
### `list_buckets`
|
||||
|
||||
List all S3 buckets accessible with the configured credentials.
|
||||
|
||||
**Parameters:** None
|
||||
|
||||
### `list_objects`
|
||||
|
||||
List objects in an S3 bucket with optional prefix filtering.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
- `bucket_name` (string): Name of the S3 bucket
|
||||
- `prefix` (string, optional): Prefix to filter objects
|
||||
|
||||
### `upload_file`
|
||||
|
||||
Upload a local file to S3 with automatic MIME type detection.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
- `local_file_path` (string): Path to the local file
|
||||
- `bucket_name` (string): Target S3 bucket
|
||||
- `object_key` (string): S3 object key (destination path)
|
||||
@@ -158,9 +219,11 @@ Upload a local file to S3 with automatic MIME type detection.
|
||||
- `cache_control` (string, optional): Cache control header
|
||||
|
||||
### `get_object`
|
||||
|
||||
Retrieve an object from S3 with two operation modes: read content directly or download to a file.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
- `bucket_name` (string): Source S3 bucket
|
||||
- `object_key` (string): S3 object key
|
||||
- `version_id` (string, optional): Version ID for versioned objects
|
||||
@@ -168,6 +231,20 @@ Retrieve an object from S3 with two operation modes: read content directly or do
|
||||
- `local_path` (string, optional): Local file path (required when mode is "download")
|
||||
- `max_content_size` (number, optional): Maximum content size in bytes for read mode (default: 1MB)
|
||||
|
||||
### `create_bucket`
|
||||
|
||||
Create a new S3 bucket with the specified name.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
- `bucket_name` (string): Source S3 bucket.
|
||||
|
||||
### `delete_bucket`
|
||||
|
||||
Delete the specified S3 bucket. If the bucket is not empty, the deletion will fail. You should delete all objects and objects inside them before calling this method.**WARNING: This operation will permanently delete the bucket and all objects within it!**
|
||||
|
||||
- `bucket_name` (string): Source S3 bucket.
|
||||
|
||||
## Architecture
|
||||
|
||||
The MCP server is built with a modular architecture:
|
||||
|
||||
@@ -151,6 +151,36 @@ impl S3Client {
|
||||
Ok(Self { client })
|
||||
}
|
||||
|
||||
pub async fn create_bucket(&self, bucket_name: &str) -> Result<BucketInfo> {
|
||||
info!("Creating S3 bucket: {}", bucket_name);
|
||||
|
||||
self.client
|
||||
.create_bucket()
|
||||
.bucket(bucket_name)
|
||||
.send()
|
||||
.await
|
||||
.context(format!("Failed to create S3 bucket: {bucket_name}"))?;
|
||||
|
||||
info!("Bucket '{}' created successfully", bucket_name);
|
||||
Ok(BucketInfo {
|
||||
name: bucket_name.to_string(),
|
||||
creation_date: None, // Creation date not returned by create_bucket
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn delete_bucket(&self, bucket_name: &str) -> Result<()> {
|
||||
info!("Deleting S3 bucket: {}", bucket_name);
|
||||
self.client
|
||||
.delete_bucket()
|
||||
.bucket(bucket_name)
|
||||
.send()
|
||||
.await
|
||||
.context(format!("Failed to delete S3 bucket: {bucket_name}"))?;
|
||||
|
||||
info!("Bucket '{}' deleted successfully", bucket_name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
debug!("Listing S3 buckets");
|
||||
|
||||
|
||||
@@ -54,6 +54,18 @@ pub struct UploadFileRequest {
|
||||
pub cache_control: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, JsonSchema)]
|
||||
pub struct CreateBucketReqeust {
|
||||
#[schemars(description = "Name of the S3 bucket to create")]
|
||||
pub bucket_name: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, JsonSchema)]
|
||||
pub struct DeleteBucketReqeust {
|
||||
#[schemars(description = "Name of the S3 bucket to delete")]
|
||||
pub bucket_name: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, JsonSchema)]
|
||||
pub struct GetObjectRequest {
|
||||
#[schemars(description = "Name of the S3 bucket")]
|
||||
@@ -110,6 +122,53 @@ impl RustfsMcpServer {
|
||||
})
|
||||
}
|
||||
|
||||
#[tool(description = "Create a new S3 bucket with the specified name")]
|
||||
pub async fn create_bucket(&self, Parameters(req): Parameters<CreateBucketReqeust>) -> String {
|
||||
info!("Executing create_bucket tool for bucket: {}", req.bucket_name);
|
||||
|
||||
match self.s3_client.create_bucket(&req.bucket_name).await {
|
||||
Ok(_) => {
|
||||
format!("Successfully created bucket: {}", req.bucket_name)
|
||||
}
|
||||
Err(e) => {
|
||||
format!("Failed to create bucket '{}': {:?}", req.bucket_name, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tool(description = "Delete an existing S3 bucket with the specified name")]
|
||||
pub async fn delete_bucket(&self, Parameters(req): Parameters<DeleteBucketReqeust>) -> String {
|
||||
info!("Executing delete_bucket tool for bucket: {}", req.bucket_name);
|
||||
|
||||
// check if bucket is empty, if not, can not delete bucket directly.
|
||||
let object_result = match self
|
||||
.s3_client
|
||||
.list_objects_v2(&req.bucket_name, ListObjectsOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(e) => {
|
||||
error!("Failed to list objects in bucket '{}': {:?}", req.bucket_name, e);
|
||||
return format!("Failed to list objects in bucket '{}': {:?}", req.bucket_name, e);
|
||||
}
|
||||
};
|
||||
|
||||
if !object_result.objects.is_empty() {
|
||||
error!("Bucket '{}' is not empty", req.bucket_name);
|
||||
return format!("Failed to delete bucket '{}': bucket is not empty", req.bucket_name);
|
||||
}
|
||||
|
||||
// delete the bucket.
|
||||
match self.s3_client.delete_bucket(&req.bucket_name).await {
|
||||
Ok(_) => {
|
||||
format!("Successfully deleted bucket: {}", req.bucket_name)
|
||||
}
|
||||
Err(e) => {
|
||||
format!("Failed to delete bucket '{}': {:?}", req.bucket_name, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tool(description = "List all S3 buckets accessible with the configured credentials")]
|
||||
pub async fn list_buckets(&self) -> String {
|
||||
info!("Executing list_buckets tool");
|
||||
@@ -667,4 +726,20 @@ mod tests {
|
||||
assert_eq!(read_mode_deser, GetObjectMode::Read);
|
||||
assert_eq!(download_mode_deser, GetObjectMode::Download);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_creation() {
|
||||
let request = CreateBucketReqeust {
|
||||
bucket_name: "test-bucket".to_string(),
|
||||
};
|
||||
assert_eq!(request.bucket_name, "test-bucket");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_deletion() {
|
||||
let request = DeleteBucketReqeust {
|
||||
bucket_name: "test-bucket".to_string(),
|
||||
};
|
||||
assert_eq!(request.bucket_name, "test-bucket");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ categories = ["web-programming", "development-tools", "filesystem"]
|
||||
documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/"
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["notify"] }
|
||||
rustfs-config = { workspace = true, features = ["notify", "constants"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["path", "sys"] }
|
||||
async-trait = { workspace = true }
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::{
|
||||
target::Target,
|
||||
};
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
use rustfs_config::notify::{ENABLE_KEY, ENABLE_ON, NOTIFY_ROUTE_PREFIX};
|
||||
use rustfs_config::notify::{ENABLE_KEY, NOTIFY_ROUTE_PREFIX};
|
||||
use rustfs_config::{DEFAULT_DELIMITER, ENV_PREFIX};
|
||||
use rustfs_ecstore::config::{Config, KVS};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
@@ -111,10 +111,10 @@ impl TargetRegistry {
|
||||
// 3.1. Instance discovery: Based on the '..._ENABLE_INSTANCEID' format
|
||||
let enable_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}_{ENABLE_KEY}_").to_uppercase();
|
||||
for (key, value) in &all_env {
|
||||
if value.eq_ignore_ascii_case(ENABLE_ON)
|
||||
|| value.eq_ignore_ascii_case("true")
|
||||
|| value.eq_ignore_ascii_case("1")
|
||||
|| value.eq_ignore_ascii_case("yes")
|
||||
if value.eq_ignore_ascii_case(rustfs_config::EnableState::One.as_str())
|
||||
|| value.eq_ignore_ascii_case(rustfs_config::EnableState::On.as_str())
|
||||
|| value.eq_ignore_ascii_case(rustfs_config::EnableState::True.as_str())
|
||||
|| value.eq_ignore_ascii_case(rustfs_config::EnableState::Yes.as_str())
|
||||
{
|
||||
if let Some(id) = key.strip_prefix(&enable_prefix) {
|
||||
if !id.is_empty() {
|
||||
@@ -202,10 +202,10 @@ impl TargetRegistry {
|
||||
let enabled = merged_config
|
||||
.lookup(ENABLE_KEY)
|
||||
.map(|v| {
|
||||
v.eq_ignore_ascii_case(ENABLE_ON)
|
||||
|| v.eq_ignore_ascii_case("true")
|
||||
|| v.eq_ignore_ascii_case("1")
|
||||
|| v.eq_ignore_ascii_case("yes")
|
||||
v.eq_ignore_ascii_case(rustfs_config::EnableState::One.as_str())
|
||||
|| v.eq_ignore_ascii_case(rustfs_config::EnableState::On.as_str())
|
||||
|| v.eq_ignore_ascii_case(rustfs_config::EnableState::True.as_str())
|
||||
|| v.eq_ignore_ascii_case(rustfs_config::EnableState::Yes.as_str())
|
||||
})
|
||||
.unwrap_or(false);
|
||||
|
||||
|
||||
@@ -556,19 +556,19 @@ mod tests {
|
||||
fn test_index_add() -> io::Result<()> {
|
||||
let mut index = Index::new();
|
||||
|
||||
// 测试添加第一个索引
|
||||
// Test adding first index
|
||||
index.add(100, 1000)?;
|
||||
assert_eq!(index.info.len(), 1);
|
||||
assert_eq!(index.info[0].compressed_offset, 100);
|
||||
assert_eq!(index.info[0].uncompressed_offset, 1000);
|
||||
|
||||
// 测试添加相同未压缩偏移量的索引
|
||||
// Test adding index with same uncompressed offset
|
||||
index.add(200, 1000)?;
|
||||
assert_eq!(index.info.len(), 1);
|
||||
assert_eq!(index.info[0].compressed_offset, 200);
|
||||
assert_eq!(index.info[0].uncompressed_offset, 1000);
|
||||
|
||||
// 测试添加新的索引(确保距离足够大)
|
||||
// Test adding new index (ensure distance is large enough)
|
||||
index.add(300, 2000 + MIN_INDEX_DIST)?;
|
||||
assert_eq!(index.info.len(), 2);
|
||||
assert_eq!(index.info[1].compressed_offset, 300);
|
||||
@@ -581,14 +581,14 @@ mod tests {
|
||||
fn test_index_add_errors() {
|
||||
let mut index = Index::new();
|
||||
|
||||
// 添加初始索引
|
||||
// Add initial index
|
||||
index.add(100, 1000).unwrap();
|
||||
|
||||
// 测试添加更小的未压缩偏移量
|
||||
// Test adding smaller uncompressed offset
|
||||
let err = index.add(200, 500).unwrap_err();
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
|
||||
// 测试添加更小的压缩偏移量
|
||||
// Test adding smaller compressed offset
|
||||
let err = index.add(50, 2000).unwrap_err();
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
}
|
||||
@@ -599,22 +599,22 @@ mod tests {
|
||||
index.total_uncompressed = 1000 + MIN_INDEX_DIST * 3;
|
||||
index.total_compressed = 5000;
|
||||
|
||||
// 添加一些测试数据,确保索引间距满足 MIN_INDEX_DIST 要求
|
||||
// Add some test data, ensure index spacing meets MIN_INDEX_DIST requirement
|
||||
index.add(100, 1000)?;
|
||||
index.add(300, 1000 + MIN_INDEX_DIST)?;
|
||||
index.add(500, 1000 + MIN_INDEX_DIST * 2)?;
|
||||
|
||||
// 测试查找存在的偏移量
|
||||
// Test finding existing offset
|
||||
let (comp, uncomp) = index.find(1500)?;
|
||||
assert_eq!(comp, 100);
|
||||
assert_eq!(uncomp, 1000);
|
||||
|
||||
// 测试查找边界值
|
||||
// Test finding boundary value
|
||||
let (comp, uncomp) = index.find(1000 + MIN_INDEX_DIST)?;
|
||||
assert_eq!(comp, 300);
|
||||
assert_eq!(uncomp, 1000 + MIN_INDEX_DIST);
|
||||
|
||||
// 测试查找最后一个索引
|
||||
// Test finding last index
|
||||
let (comp, uncomp) = index.find(1000 + MIN_INDEX_DIST * 2)?;
|
||||
assert_eq!(comp, 500);
|
||||
assert_eq!(uncomp, 1000 + MIN_INDEX_DIST * 2);
|
||||
@@ -628,16 +628,16 @@ mod tests {
|
||||
index.total_uncompressed = 10000;
|
||||
index.total_compressed = 5000;
|
||||
|
||||
// 测试未初始化的索引
|
||||
// Test uninitialized index
|
||||
let uninit_index = Index::new();
|
||||
let err = uninit_index.find(1000).unwrap_err();
|
||||
assert_eq!(err.kind(), io::ErrorKind::Other);
|
||||
|
||||
// 测试超出范围的偏移量
|
||||
// Test offset out of range
|
||||
let err = index.find(15000).unwrap_err();
|
||||
assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof);
|
||||
|
||||
// 测试负数偏移量
|
||||
// Test negative offset
|
||||
let err = match index.find(-1000) {
|
||||
Ok(_) => panic!("should be error"),
|
||||
Err(e) => e,
|
||||
@@ -650,15 +650,15 @@ mod tests {
|
||||
let mut index = Index::new();
|
||||
index.est_block_uncomp = MIN_INDEX_DIST;
|
||||
|
||||
// 添加超过最大索引数量的条目,确保间距满足 MIN_INDEX_DIST 要求
|
||||
// Add entries exceeding maximum index count, ensure spacing meets MIN_INDEX_DIST requirement
|
||||
for i in 0..MAX_INDEX_ENTRIES + 100 {
|
||||
index.add(i as i64 * 100, i as i64 * MIN_INDEX_DIST).unwrap();
|
||||
}
|
||||
|
||||
// 手动调用 reduce 方法
|
||||
// Manually call reduce method
|
||||
index.reduce();
|
||||
|
||||
// 验证索引数量是否被正确减少
|
||||
// Verify index count has been correctly reduced
|
||||
assert!(index.info.len() <= MAX_INDEX_ENTRIES);
|
||||
}
|
||||
|
||||
@@ -666,16 +666,16 @@ mod tests {
|
||||
fn test_index_json() -> io::Result<()> {
|
||||
let mut index = Index::new();
|
||||
|
||||
// 添加一些测试数据
|
||||
// Add some test data
|
||||
index.add(100, 1000)?;
|
||||
index.add(300, 2000 + MIN_INDEX_DIST)?;
|
||||
|
||||
// 测试 JSON 序列化
|
||||
// Test JSON serialization
|
||||
let json = index.to_json().unwrap();
|
||||
let json_str = String::from_utf8(json).unwrap();
|
||||
|
||||
println!("json_str: {json_str}");
|
||||
// 验证 JSON 内容
|
||||
// Verify JSON content
|
||||
|
||||
assert!(json_str.contains("\"compressed\": 100"));
|
||||
assert!(json_str.contains("\"uncompressed\": 1000"));
|
||||
|
||||
@@ -443,7 +443,7 @@ mod tests {
|
||||
let mut compressed = Vec::new();
|
||||
compress_reader.read_to_end(&mut compressed).await.unwrap();
|
||||
|
||||
// DecompressReader解包
|
||||
// DecompressReader unpacking
|
||||
let mut decompress_reader = DecompressReader::new(Cursor::new(compressed.clone()), CompressionAlgorithm::Gzip);
|
||||
let mut decompressed = Vec::new();
|
||||
decompress_reader.read_to_end(&mut decompressed).await.unwrap();
|
||||
@@ -460,7 +460,7 @@ mod tests {
|
||||
let mut compressed = Vec::new();
|
||||
compress_reader.read_to_end(&mut compressed).await.unwrap();
|
||||
|
||||
// DecompressReader解包
|
||||
// DecompressReader unpacking
|
||||
let mut decompress_reader = DecompressReader::new(Cursor::new(compressed.clone()), CompressionAlgorithm::Deflate);
|
||||
let mut decompressed = Vec::new();
|
||||
decompress_reader.read_to_end(&mut decompressed).await.unwrap();
|
||||
|
||||
@@ -223,7 +223,7 @@ mod tests {
|
||||
let n = etag_reader.read_to_end(&mut buf).await.unwrap();
|
||||
assert_eq!(n, data.len());
|
||||
assert_eq!(&buf, data);
|
||||
// 校验通过,etag应等于expected
|
||||
// Verification passed, etag should equal expected
|
||||
assert_eq!(etag_reader.try_resolve_etag(), Some(expected));
|
||||
}
|
||||
|
||||
@@ -236,7 +236,7 @@ mod tests {
|
||||
let mut etag_reader = EtagReader::new(reader, Some(wrong_checksum));
|
||||
|
||||
let mut buf = Vec::new();
|
||||
// 校验失败,应该返回InvalidData错误
|
||||
// Verification failed, should return InvalidData error
|
||||
let err = etag_reader.read_to_end(&mut buf).await.unwrap_err();
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
@@ -123,7 +123,7 @@ mod tests {
|
||||
let hardlimit = HardLimitReader::new(reader, 3);
|
||||
let mut r = hardlimit;
|
||||
let mut buf = vec![0u8; 10];
|
||||
// 读取超限,应该返回错误
|
||||
// Reading exceeds limit, should return error
|
||||
let err = match read_full(&mut r, &mut buf).await {
|
||||
Ok(n) => {
|
||||
println!("Read {n} bytes");
|
||||
|
||||
@@ -53,9 +53,9 @@ pub trait QueryExecution: Send + Sync {
|
||||
fn query_type(&self) -> QueryType {
|
||||
QueryType::Batch
|
||||
}
|
||||
// 开始
|
||||
// Start
|
||||
async fn start(&self) -> QueryResult<Output>;
|
||||
// 停止
|
||||
// Stop
|
||||
fn cancel(&self) -> QueryResult<()>;
|
||||
}
|
||||
|
||||
|
||||
@@ -103,7 +103,7 @@ impl TableSource for TableSourceAdapter {
|
||||
}
|
||||
|
||||
/// Called by [`InlineTableScan`]
|
||||
fn get_logical_plan(&self) -> Option<Cow<LogicalPlan>> {
|
||||
fn get_logical_plan(&self) -> Option<Cow<'_, LogicalPlan>> {
|
||||
Some(Cow::Owned(self.plan.clone()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,15 +87,15 @@ impl PhysicalPlanner for DefaultPhysicalPlanner {
|
||||
logical_plan: &LogicalPlan,
|
||||
session: &SessionCtx,
|
||||
) -> QueryResult<Arc<dyn ExecutionPlan>> {
|
||||
// 将扩展的物理计划优化规则注入 df 的 session state
|
||||
// Inject extended physical plan optimization rules into df's session state
|
||||
let new_state = SessionStateBuilder::new_from_existing(session.inner().clone())
|
||||
.with_physical_optimizer_rules(self.ext_physical_optimizer_rules.clone())
|
||||
.build();
|
||||
|
||||
// 通过扩展的物理计划转换规则构造 df 的 Physical Planner
|
||||
// Construct df's Physical Planner with extended physical plan transformation rules
|
||||
let planner = DFDefaultPhysicalPlanner::with_extension_planners(self.ext_physical_transform_rules.clone());
|
||||
|
||||
// 执行 df 的物理计划规划及优化
|
||||
// Execute df's physical plan planning and optimization
|
||||
planner
|
||||
.create_physical_plan(logical_plan, &new_state)
|
||||
.await
|
||||
|
||||
@@ -31,14 +31,11 @@ bytes = { workspace = true }
|
||||
http.workspace = true
|
||||
time.workspace = true
|
||||
hyper.workspace = true
|
||||
serde.workspace = true
|
||||
serde_urlencoded.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["full"] }
|
||||
s3s.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -21,7 +21,7 @@ use std::collections::HashMap;
|
||||
use std::io::Error;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::{fs, io};
|
||||
use std::{env, fs, io};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
/// Load public certificate from file.
|
||||
@@ -194,6 +194,19 @@ pub fn create_multi_cert_resolver(
|
||||
})
|
||||
}
|
||||
|
||||
/// Checks if TLS key logging is enabled.
|
||||
pub fn tls_key_log() -> bool {
|
||||
env::var("RUSTFS_TLS_KEYLOG")
|
||||
.map(|v| {
|
||||
let v = v.trim();
|
||||
v.eq_ignore_ascii_case("1")
|
||||
|| v.eq_ignore_ascii_case("on")
|
||||
|| v.eq_ignore_ascii_case("true")
|
||||
|| v.eq_ignore_ascii_case("yes")
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -289,44 +289,44 @@ mod tests {
|
||||
CompressionAlgorithm::Snappy,
|
||||
];
|
||||
|
||||
println!("\n压缩算法基准测试结果:");
|
||||
println!("\nCompression algorithm benchmark results:");
|
||||
println!(
|
||||
"{:<10} {:<10} {:<15} {:<15} {:<15}",
|
||||
"数据大小", "算法", "压缩时间(ms)", "压缩后大小", "压缩率"
|
||||
"Data Size", "Algorithm", "Compress Time(ms)", "Compressed Size", "Compression Ratio"
|
||||
);
|
||||
|
||||
for size in sizes {
|
||||
// 生成可压缩的数据(重复的文本模式)
|
||||
// Generate compressible data (repeated text pattern)
|
||||
let pattern = b"Hello, this is a test pattern that will be repeated multiple times to create compressible data. ";
|
||||
let data: Vec<u8> = pattern.iter().cycle().take(size).copied().collect();
|
||||
|
||||
for algo in algorithms {
|
||||
// 压缩测试
|
||||
// Compression test
|
||||
let start = Instant::now();
|
||||
let compressed = compress_block(&data, algo);
|
||||
let compress_time = start.elapsed();
|
||||
let compression_time = start.elapsed();
|
||||
|
||||
// 解压测试
|
||||
// Decompression test
|
||||
let start = Instant::now();
|
||||
let _decompressed = decompress_block(&compressed, algo).unwrap();
|
||||
let _decompress_time = start.elapsed();
|
||||
let _decompression_time = start.elapsed();
|
||||
|
||||
// 计算压缩率
|
||||
// Calculate compression ratio
|
||||
let compression_ratio = (size as f64 / compressed.len() as f64) as f32;
|
||||
|
||||
println!(
|
||||
"{:<10} {:<10} {:<15.2} {:<15} {:<15.2}x",
|
||||
format!("{}KB", size / 1024),
|
||||
algo.as_str(),
|
||||
compress_time.as_secs_f64() * 1000.0,
|
||||
compression_time.as_secs_f64() * 1000.0,
|
||||
compressed.len(),
|
||||
compression_ratio
|
||||
);
|
||||
|
||||
// 验证解压结果
|
||||
// Verify decompression result
|
||||
assert_eq!(_decompressed, data);
|
||||
}
|
||||
println!(); // 添加空行分隔不同大小的结果
|
||||
println!(); // Add blank line to separate results of different sizes
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,18 +110,18 @@ use siphasher::sip::SipHasher;
|
||||
pub const EMPTY_STRING_SHA256_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
||||
|
||||
pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize {
|
||||
// 你的密钥,必须是 16 字节
|
||||
// Your key, must be 16 bytes
|
||||
|
||||
// 计算字符串的 SipHash 值
|
||||
// Calculate SipHash value of the string
|
||||
let result = SipHasher::new_with_key(id).hash(key.as_bytes());
|
||||
|
||||
result as usize % cardinality
|
||||
(result as usize) % cardinality
|
||||
}
|
||||
|
||||
pub fn crc_hash(key: &str, cardinality: usize) -> usize {
|
||||
let mut hasher = Hasher::new(); // 创建一个新的哈希器
|
||||
let mut hasher = Hasher::new(); // Create a new hasher
|
||||
|
||||
hasher.update(key.as_bytes()); // 更新哈希状态,添加数据
|
||||
hasher.update(key.as_bytes()); // Update hash state, add data
|
||||
|
||||
let checksum = hasher.finalize();
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ pub mod net;
|
||||
#[cfg(feature = "net")]
|
||||
pub use net::*;
|
||||
|
||||
#[cfg(all(feature = "net", feature = "io"))]
|
||||
pub mod retry;
|
||||
|
||||
#[cfg(feature = "io")]
|
||||
|
||||
+26
-26
@@ -599,7 +599,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_format_consistency_with_extensions() {
|
||||
// 测试格式与扩展名的一致性
|
||||
// Test format consistency with extensions
|
||||
let consistency_tests = vec![
|
||||
(CompressionFormat::Gzip, "gz"),
|
||||
(CompressionFormat::Bzip2, "bz2"),
|
||||
@@ -721,20 +721,20 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_compression_format_clone_and_copy() {
|
||||
// 测试 CompressionFormat 是否可以被复制
|
||||
// Test if CompressionFormat can be copied
|
||||
let format = CompressionFormat::Gzip;
|
||||
let format_copy = format;
|
||||
|
||||
// 验证复制后的值相等
|
||||
// Verify copied values are equal
|
||||
assert_eq!(format, format_copy);
|
||||
|
||||
// 验证原值仍然可用
|
||||
// Verify original value is still usable
|
||||
assert_eq!(format, CompressionFormat::Gzip);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compression_format_match_exhaustiveness() {
|
||||
// 测试 match 语句的完整性
|
||||
// Test match statement completeness
|
||||
fn handle_format(format: CompressionFormat) -> &'static str {
|
||||
match format {
|
||||
CompressionFormat::Gzip => "gzip",
|
||||
@@ -748,7 +748,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// 测试所有变体都有对应的处理
|
||||
// Test all variants have corresponding handlers
|
||||
assert_eq!(handle_format(CompressionFormat::Gzip), "gzip");
|
||||
assert_eq!(handle_format(CompressionFormat::Bzip2), "bzip2");
|
||||
assert_eq!(handle_format(CompressionFormat::Zip), "zip");
|
||||
@@ -760,10 +760,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extension_parsing_performance() {
|
||||
// 测试扩展名解析的性能(简单的性能测试)
|
||||
// Test extension parsing performance (simple performance test)
|
||||
let extensions = vec!["gz", "bz2", "zip", "xz", "zlib", "zst", "unknown"];
|
||||
|
||||
// 多次调用以测试性能一致性
|
||||
// Multiple calls to test performance consistency
|
||||
for _ in 0..1000 {
|
||||
for ext in &extensions {
|
||||
let _format = CompressionFormat::from_extension(ext);
|
||||
@@ -775,7 +775,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_format_default_behavior() {
|
||||
// 测试格式的默认行为
|
||||
// Test format default behavior
|
||||
let unknown_extensions = vec!["", "txt", "doc", "pdf", "unknown_ext"];
|
||||
|
||||
for ext in unknown_extensions {
|
||||
@@ -786,7 +786,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_compression_level() {
|
||||
// 测试压缩级别
|
||||
// Test compression level
|
||||
let default_level = CompressionLevel::default();
|
||||
assert_eq!(default_level, CompressionLevel::Default);
|
||||
|
||||
@@ -800,7 +800,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_format_extension() {
|
||||
// 测试格式扩展名获取
|
||||
// Test format extension retrieval
|
||||
assert_eq!(CompressionFormat::Gzip.extension(), "gz");
|
||||
assert_eq!(CompressionFormat::Bzip2.extension(), "bz2");
|
||||
assert_eq!(CompressionFormat::Zip.extension(), "zip");
|
||||
@@ -813,7 +813,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_format_is_supported() {
|
||||
// 测试格式支持检查
|
||||
// Test format support check
|
||||
assert!(CompressionFormat::Gzip.is_supported());
|
||||
assert!(CompressionFormat::Bzip2.is_supported());
|
||||
assert!(CompressionFormat::Zip.is_supported());
|
||||
@@ -826,7 +826,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_format_from_path() {
|
||||
// 测试从路径识别格式
|
||||
// Test format recognition from path
|
||||
use std::path::Path;
|
||||
|
||||
assert_eq!(CompressionFormat::from_path("file.gz"), CompressionFormat::Gzip);
|
||||
@@ -840,7 +840,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_encoder_supported_formats() {
|
||||
// 测试支持的格式能够创建编码器
|
||||
// Test supported formats can create encoders
|
||||
use std::io::Cursor;
|
||||
|
||||
let output = Vec::new();
|
||||
@@ -853,7 +853,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_encoder_unsupported_formats() {
|
||||
// 测试不支持的格式返回错误
|
||||
// Test unsupported formats return errors
|
||||
use std::io::Cursor;
|
||||
|
||||
let output1 = Vec::new();
|
||||
@@ -900,7 +900,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_decompressor_creation() {
|
||||
// 测试解压缩器创建
|
||||
// Test decompressor creation
|
||||
let decompressor = Decompressor::new(CompressionFormat::Gzip);
|
||||
assert_eq!(decompressor.format, CompressionFormat::Gzip);
|
||||
|
||||
@@ -910,7 +910,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_zip_entry_creation() {
|
||||
// 测试 ZIP 条目信息创建
|
||||
// Test ZIP entry info creation
|
||||
let entry = ZipEntry {
|
||||
name: "test.txt".to_string(),
|
||||
size: 1024,
|
||||
@@ -928,7 +928,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_compression_level_variants() {
|
||||
// 测试压缩级别的所有变体
|
||||
// Test all compression level variants
|
||||
let levels = vec![
|
||||
CompressionLevel::Fastest,
|
||||
CompressionLevel::Best,
|
||||
@@ -938,14 +938,14 @@ mod tests {
|
||||
];
|
||||
|
||||
for level in levels {
|
||||
// 验证每个级别都有对应的 Debug 实现
|
||||
// Verify each level has corresponding Debug implementation
|
||||
let _debug_str = format!("{level:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_comprehensive_coverage() {
|
||||
// 测试格式的全面覆盖
|
||||
// Test comprehensive format coverage
|
||||
let all_formats = vec![
|
||||
CompressionFormat::Gzip,
|
||||
CompressionFormat::Bzip2,
|
||||
@@ -958,13 +958,13 @@ mod tests {
|
||||
];
|
||||
|
||||
for format in all_formats {
|
||||
// 验证每个格式都有扩展名
|
||||
// Verify each format has an extension
|
||||
let _ext = format.extension();
|
||||
|
||||
// 验证支持状态检查
|
||||
// Verify support status check
|
||||
let _supported = format.is_supported();
|
||||
|
||||
// 验证 Debug 实现
|
||||
// Verify Debug implementation
|
||||
let _debug = format!("{format:?}");
|
||||
}
|
||||
}
|
||||
@@ -975,7 +975,7 @@ mod tests {
|
||||
// use std::path::Path;
|
||||
// use tokio::fs::File;
|
||||
|
||||
// let input_path = "/Users/weisd/Downloads/wsd.tar.gz"; // 替换为你的压缩文件路径
|
||||
// let input_path = "/Users/weisd/Downloads/wsd.tar.gz"; // Replace with your compressed file path
|
||||
|
||||
// let f = File::open(input_path).await?;
|
||||
|
||||
@@ -994,8 +994,8 @@ mod tests {
|
||||
// )
|
||||
// .await
|
||||
// {
|
||||
// Ok(_) => println!("解压成功!"),
|
||||
// Err(e) => println!("解压失败:{}", e),
|
||||
// Ok(_) => println!("Decompression successful!"),
|
||||
// Err(e) => println!("Decompression failed: {}", e),
|
||||
// }
|
||||
|
||||
// Ok(())
|
||||
|
||||
+11
-11
@@ -1,22 +1,22 @@
|
||||
# RustFS 管理员用户名
|
||||
# RustFS admin username
|
||||
RUSTFS_ROOT_USER=rustfsadmin
|
||||
# RustFS 管理员密码
|
||||
# RustFS admin password
|
||||
RUSTFS_ROOT_PASSWORD=rustfsadmin
|
||||
|
||||
# 数据卷配置示例路径:deploy/data/rustfs.env
|
||||
# RustFS 数据卷存储路径,支持多卷配置,vol1 到 vol4
|
||||
# Data volume configuration example path: deploy/data/rustfs.env
|
||||
# RustFS data volume storage path, supports multi-volume configuration, vol1 to vol4
|
||||
RUSTFS_VOLUMES="./deploy/deploy/vol{1...4}"
|
||||
# RustFS 服务启动参数,指定监听地址和端口
|
||||
# RustFS service startup parameters, specify listening address and port
|
||||
RUSTFS_OPTS="--address :9000"
|
||||
# RustFS 服务监听地址和端口
|
||||
# RustFS service listening address and port
|
||||
RUSTFS_ADDRESS=":9000"
|
||||
# 是否启用 RustFS 控制台功能
|
||||
# Whether to enable RustFS console functionality
|
||||
RUSTFS_CONSOLE_ENABLE=true
|
||||
# RustFS 服务域名配置
|
||||
# RustFS service domain configuration
|
||||
RUSTFS_SERVER_DOMAINS=127.0.0.1:9000
|
||||
# RustFS 许可证内容
|
||||
# RustFS license content
|
||||
RUSTFS_LICENSE="license content"
|
||||
# 可观测性配置Endpoint:http://localhost:4317
|
||||
# Observability configuration Endpoint: http://localhost:4317
|
||||
RUSTFS_OBS_ENDPOINT=http://localhost:4317
|
||||
# TLS 证书目录路径:deploy/certs
|
||||
# TLS certificate directory path: deploy/certs
|
||||
RUSTFS_TLS_PATH=/etc/default/tls
|
||||
+3
-11
@@ -150,11 +150,7 @@ build_and_push() {
|
||||
else
|
||||
print_message $RED "❌ Failed to build latest variant"
|
||||
print_message $YELLOW "💡 Note: Make sure rustfs binaries are available at:"
|
||||
if [ "$CHANNEL" = "dev" ]; then
|
||||
print_message $YELLOW " https://dl.rustfs.com/artifacts/rustfs/dev/"
|
||||
else
|
||||
print_message $YELLOW " https://dl.rustfs.com/artifacts/rustfs/release/"
|
||||
fi
|
||||
print_message $YELLOW " https://github.com/rustfs/rustfs/releases"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -183,11 +179,7 @@ build_and_push() {
|
||||
else
|
||||
print_message $RED "❌ Failed to build release variant"
|
||||
print_message $YELLOW "💡 Note: Make sure rustfs binaries are available at:"
|
||||
if [ "$CHANNEL" = "dev" ]; then
|
||||
print_message $YELLOW " https://dl.rustfs.com/artifacts/rustfs/dev/"
|
||||
else
|
||||
print_message $YELLOW " https://dl.rustfs.com/artifacts/rustfs/release/"
|
||||
fi
|
||||
print_message $YELLOW " https://github.com/rustfs/rustfs/releases"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
@@ -248,7 +240,7 @@ done
|
||||
# Main execution
|
||||
main() {
|
||||
print_message $BLUE "🐳 RustFS Docker Buildx Build Script"
|
||||
print_message $YELLOW "📋 Build Strategy: Uses pre-built binaries from dl.rustfs.com"
|
||||
print_message $YELLOW "📋 Build Strategy: Uses pre-built binaries from GitHub Releases"
|
||||
print_message $YELLOW "🚀 Production images only - optimized for distribution"
|
||||
echo ""
|
||||
|
||||
|
||||
@@ -54,8 +54,6 @@ rustfs-s3select-query = { workspace = true }
|
||||
atoi = { workspace = true }
|
||||
atomic_enum = { workspace = true }
|
||||
axum.workspace = true
|
||||
axum-extra = { workspace = true }
|
||||
axum-server = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
|
||||
@@ -53,7 +53,6 @@ use s3s::stream::{ByteStream, DynByteStream};
|
||||
use s3s::{Body, S3Error, S3Request, S3Response, S3Result, s3_error};
|
||||
use s3s::{S3ErrorCode, StdError};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::debug;
|
||||
// use serde_json::to_vec;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
@@ -65,6 +64,7 @@ use tokio::sync::mpsc::{self};
|
||||
use tokio::time::interval;
|
||||
use tokio::{select, spawn};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tracing::debug;
|
||||
use tracing::{error, info, warn};
|
||||
// use url::UrlQuery;
|
||||
|
||||
@@ -81,6 +81,7 @@ pub mod trace;
|
||||
pub mod user;
|
||||
use urlencoding::decode;
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Serialize, Default)]
|
||||
#[serde(rename_all = "PascalCase", default)]
|
||||
pub struct AccountInfo {
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::admin::router::Operation;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use http::{HeaderMap, StatusCode};
|
||||
|
||||
@@ -341,6 +341,7 @@ impl Operation for RemoveTier {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct VerifyTier {}
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for VerifyTier {
|
||||
|
||||
@@ -22,6 +22,7 @@ use tracing::warn;
|
||||
|
||||
use crate::admin::router::Operation;
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn extract_trace_options(uri: &Uri) -> S3Result<ServiceTraceOpts> {
|
||||
let mut st_opts = ServiceTraceOpts::default();
|
||||
st_opts
|
||||
@@ -31,6 +32,7 @@ fn extract_trace_options(uri: &Uri) -> S3Result<ServiceTraceOpts> {
|
||||
Ok(st_opts)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct Trace {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
|
||||
+2
-4
@@ -490,11 +490,9 @@ mod tests {
|
||||
let result = check_claims_from_token("", &cred);
|
||||
|
||||
// This might fail due to global state dependencies, but should return error about global cred init
|
||||
if result.is_ok() {
|
||||
let claims = result.unwrap();
|
||||
if let Ok(claims) = result {
|
||||
assert!(claims.is_empty());
|
||||
} else {
|
||||
let error = result.unwrap_err();
|
||||
} else if let Err(error) = result {
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
assert!(error.message().unwrap_or("").contains("action cred not init"));
|
||||
}
|
||||
|
||||
@@ -183,10 +183,6 @@ async fn run(opt: config::Opt) -> Result<()> {
|
||||
Error::other(err)
|
||||
})?;
|
||||
|
||||
// init scanner and auto heal with unified cancellation token
|
||||
// let _background_services_cancel_token = create_background_services_cancel_token();
|
||||
// init_data_scanner().await;
|
||||
// init_auto_heal().await;
|
||||
let _ = create_ahm_services_cancel_token();
|
||||
|
||||
// Initialize heal manager with channel processor
|
||||
|
||||
+72
-14
@@ -14,7 +14,6 @@
|
||||
|
||||
// Ensure the correct path for parse_license is imported
|
||||
use crate::admin;
|
||||
// use crate::admin::console::{CONSOLE_CONFIG, init_console_cfg};
|
||||
use crate::auth::IAMAuth;
|
||||
use crate::config;
|
||||
use crate::server::hybrid::hybrid;
|
||||
@@ -43,8 +42,6 @@ use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
#[cfg(unix)]
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
use tokio_rustls::TlsAcceptor;
|
||||
use tonic::{Request, Status, metadata::MetadataValue};
|
||||
use tower::ServiceBuilder;
|
||||
@@ -63,9 +60,6 @@ pub async fn start_http_server(
|
||||
let server_port = server_addr.port();
|
||||
let server_address = server_addr.to_string();
|
||||
|
||||
// The listening address and port are obtained from the parameters
|
||||
// let listener = TcpListener::bind(server_address.clone()).await?;
|
||||
|
||||
// The listening address and port are obtained from the parameters
|
||||
let listener = {
|
||||
let mut server_addr = server_addr;
|
||||
@@ -172,6 +166,7 @@ pub async fn start_http_server(
|
||||
tokio::spawn(async move {
|
||||
#[cfg(unix)]
|
||||
let (mut sigterm_inner, mut sigint_inner) = {
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
// Unix platform specific code
|
||||
let sigterm_inner = signal(SignalKind::terminate()).expect("Failed to create SIGTERM signal handler");
|
||||
let sigint_inner = signal(SignalKind::interrupt()).expect("Failed to create SIGINT signal handler");
|
||||
@@ -292,32 +287,55 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
|
||||
|
||||
debug!("Found TLS directory, checking for certificates");
|
||||
|
||||
// 1. Try to load all certificates from the directory (multi-cert support)
|
||||
// Make sure to use a modern encryption suite
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
// 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI)
|
||||
if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path) {
|
||||
if !cert_key_pairs.is_empty() {
|
||||
debug!("Found {} certificates, creating multi-cert resolver", cert_key_pairs.len());
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
debug!("Found {} certificates, creating SNI-aware multi-cert resolver", cert_key_pairs.len());
|
||||
|
||||
// Create an SNI-enabled certificate resolver
|
||||
let resolver = rustfs_utils::create_multi_cert_resolver(cert_key_pairs)?;
|
||||
|
||||
// Configure the server to enable SNI support
|
||||
let mut server_config = ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_cert_resolver(Arc::new(rustfs_utils::create_multi_cert_resolver(cert_key_pairs)?));
|
||||
.with_cert_resolver(Arc::new(resolver));
|
||||
|
||||
// Configure ALPN protocol priority
|
||||
server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];
|
||||
|
||||
// Log SNI requests
|
||||
if rustfs_utils::tls_key_log() {
|
||||
server_config.key_log = Arc::new(rustls::KeyLogFile::new());
|
||||
}
|
||||
|
||||
return Ok(Some(TlsAcceptor::from(Arc::new(server_config))));
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Fallback to legacy single certificate mode
|
||||
// 2. Revert to the traditional single-certificate mode
|
||||
let key_path = format!("{tls_path}/{RUSTFS_TLS_KEY}");
|
||||
let cert_path = format!("{tls_path}/{RUSTFS_TLS_CERT}");
|
||||
if tokio::try_join!(tokio::fs::metadata(&key_path), tokio::fs::metadata(&cert_path)).is_ok() {
|
||||
debug!("Found legacy single TLS certificate, starting with HTTPS");
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
let certs = rustfs_utils::load_certs(&cert_path).map_err(|e| rustfs_utils::certs_error(e.to_string()))?;
|
||||
let key = rustfs_utils::load_private_key(&key_path).map_err(|e| rustfs_utils::certs_error(e.to_string()))?;
|
||||
|
||||
let mut server_config = ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_single_cert(certs, key)
|
||||
.map_err(|e| rustfs_utils::certs_error(e.to_string()))?;
|
||||
|
||||
// Configure ALPN protocol priority
|
||||
server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];
|
||||
|
||||
// Log SNI requests
|
||||
if rustfs_utils::tls_key_log() {
|
||||
server_config.key_log = Arc::new(rustls::KeyLogFile::new());
|
||||
}
|
||||
|
||||
return Ok(Some(TlsAcceptor::from(Arc::new(server_config))));
|
||||
}
|
||||
|
||||
@@ -398,6 +416,10 @@ fn process_connection(
|
||||
// Decide whether to handle HTTPS or HTTP connections based on the existence of TLS Acceptor
|
||||
if let Some(acceptor) = tls_acceptor {
|
||||
debug!("TLS handshake start");
|
||||
let peer_addr = socket
|
||||
.peer_addr()
|
||||
.ok()
|
||||
.map_or_else(|| "unknown".to_string(), |addr| addr.to_string());
|
||||
match acceptor.accept(socket).await {
|
||||
Ok(tls_socket) => {
|
||||
debug!("TLS handshake successful");
|
||||
@@ -408,8 +430,44 @@ fn process_connection(
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
error!(?err, "TLS handshake failed");
|
||||
return; // Failed to end the task directly
|
||||
// Detailed analysis of the reasons why the TLS handshake fails
|
||||
let err_str = err.to_string();
|
||||
let mut key_failure_type_str: &str = "UNKNOWN";
|
||||
if err_str.contains("unexpected EOF") || err_str.contains("handshake eof") {
|
||||
warn!(peer_addr = %peer_addr, "TLS handshake failed. If this client needs HTTP, it should connect to the HTTP port instead");
|
||||
key_failure_type_str = "UNEXPECTED_EOF";
|
||||
} else if err_str.contains("protocol version") {
|
||||
error!(
|
||||
peer_addr = %peer_addr,
|
||||
"TLS handshake failed due to protocol version mismatch: {}", err
|
||||
);
|
||||
key_failure_type_str = "PROTOCOL_VERSION";
|
||||
} else if err_str.contains("certificate") {
|
||||
error!(
|
||||
peer_addr = %peer_addr,
|
||||
"TLS handshake failed due to certificate issues: {}", err
|
||||
);
|
||||
key_failure_type_str = "CERTIFICATE";
|
||||
} else {
|
||||
error!(
|
||||
peer_addr = %peer_addr,
|
||||
"TLS handshake failed: {}", err
|
||||
);
|
||||
}
|
||||
info!(
|
||||
counter.rustfs_tls_handshake_failures = 1_u64,
|
||||
key_failure_type = key_failure_type_str,
|
||||
"TLS handshake failure metric"
|
||||
);
|
||||
// Record detailed diagnostic information
|
||||
debug!(
|
||||
peer_addr = %peer_addr,
|
||||
error_type = %std::any::type_name_of_val(&err),
|
||||
error_details = %err,
|
||||
"TLS handshake failure details"
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
debug!("TLS handshake success");
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::error::ApiError;
|
||||
use crate::storage::access::ReqInfo;
|
||||
use crate::storage::options::copy_dst_opts;
|
||||
use crate::storage::options::copy_src_opts;
|
||||
use crate::storage::options::{extract_metadata_from_mime, get_opts};
|
||||
use crate::storage::options::{extract_metadata_from_mime_with_object_name, get_opts};
|
||||
use bytes::Bytes;
|
||||
use chrono::DateTime;
|
||||
use chrono::Utc;
|
||||
@@ -311,7 +311,7 @@ impl S3 for FS {
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
force_create: true,
|
||||
force_create: false, // TODO: force support
|
||||
lock_enabled: object_lock_enabled_for_bucket.is_some_and(|v| v),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -1318,7 +1318,7 @@ impl S3 for FS {
|
||||
let objects: Vec<ObjectVersion> = object_infos
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|v| !v.name.is_empty())
|
||||
.filter(|v| !v.name.is_empty() && !v.delete_marker)
|
||||
.map(|v| {
|
||||
ObjectVersion {
|
||||
key: Some(v.name.to_owned()),
|
||||
@@ -1340,6 +1340,19 @@ impl S3 for FS {
|
||||
.map(|v| CommonPrefix { prefix: Some(v) })
|
||||
.collect();
|
||||
|
||||
let delete_markers = object_infos
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|o| o.delete_marker)
|
||||
.map(|o| DeleteMarkerEntry {
|
||||
key: Some(o.name.clone()),
|
||||
version_id: o.version_id.map(|v| v.to_string()),
|
||||
is_latest: Some(o.is_latest),
|
||||
last_modified: o.mod_time.map(Timestamp::from),
|
||||
..Default::default()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let output = ListObjectVersionsOutput {
|
||||
// is_truncated: Some(object_infos.is_truncated),
|
||||
max_keys: Some(key_count),
|
||||
@@ -1348,6 +1361,7 @@ impl S3 for FS {
|
||||
prefix: Some(prefix),
|
||||
common_prefixes: Some(common_prefixes),
|
||||
versions: Some(objects),
|
||||
delete_markers: Some(delete_markers),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -1412,7 +1426,7 @@ impl S3 for FS {
|
||||
|
||||
let mut metadata = metadata.unwrap_or_default();
|
||||
|
||||
extract_metadata_from_mime(&req.headers, &mut metadata);
|
||||
extract_metadata_from_mime_with_object_name(&req.headers, &mut metadata, Some(&key));
|
||||
|
||||
if let Some(tags) = tagging {
|
||||
metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags);
|
||||
|
||||
@@ -186,6 +186,15 @@ pub fn extract_metadata(headers: &HeaderMap<HeaderValue>) -> HashMap<String, Str
|
||||
|
||||
/// Extracts metadata from headers and returns it as a HashMap.
|
||||
pub fn extract_metadata_from_mime(headers: &HeaderMap<HeaderValue>, metadata: &mut HashMap<String, String>) {
|
||||
extract_metadata_from_mime_with_object_name(headers, metadata, None);
|
||||
}
|
||||
|
||||
/// Extracts metadata from headers and returns it as a HashMap with object name for MIME type detection.
|
||||
pub fn extract_metadata_from_mime_with_object_name(
|
||||
headers: &HeaderMap<HeaderValue>,
|
||||
metadata: &mut HashMap<String, String>,
|
||||
object_name: Option<&str>,
|
||||
) {
|
||||
for (k, v) in headers.iter() {
|
||||
if let Some(key) = k.as_str().strip_prefix("x-amz-meta-") {
|
||||
if key.is_empty() {
|
||||
@@ -210,10 +219,42 @@ pub fn extract_metadata_from_mime(headers: &HeaderMap<HeaderValue>, metadata: &m
|
||||
}
|
||||
|
||||
if !metadata.contains_key("content-type") {
|
||||
metadata.insert("content-type".to_owned(), "binary/octet-stream".to_owned());
|
||||
let default_content_type = if let Some(obj_name) = object_name {
|
||||
detect_content_type_from_object_name(obj_name)
|
||||
} else {
|
||||
"binary/octet-stream".to_owned()
|
||||
};
|
||||
metadata.insert("content-type".to_owned(), default_content_type);
|
||||
}
|
||||
}
|
||||
|
||||
/// Detects content type from object name based on file extension.
|
||||
pub(crate) fn detect_content_type_from_object_name(object_name: &str) -> String {
|
||||
let lower_name = object_name.to_lowercase();
|
||||
|
||||
// Check for Parquet files specifically
|
||||
if lower_name.ends_with(".parquet") {
|
||||
return "application/vnd.apache.parquet".to_owned();
|
||||
}
|
||||
|
||||
// Special handling for other data formats that mime_guess doesn't know
|
||||
if lower_name.ends_with(".avro") {
|
||||
return "application/avro".to_owned();
|
||||
}
|
||||
if lower_name.ends_with(".orc") {
|
||||
return "application/orc".to_owned();
|
||||
}
|
||||
if lower_name.ends_with(".feather") {
|
||||
return "application/feather".to_owned();
|
||||
}
|
||||
if lower_name.ends_with(".arrow") {
|
||||
return "application/arrow".to_owned();
|
||||
}
|
||||
|
||||
// Use mime_guess for standard file types
|
||||
mime_guess::from_path(object_name).first_or_octet_stream().to_string()
|
||||
}
|
||||
|
||||
/// List of supported headers.
|
||||
static SUPPORTED_HEADERS: LazyLock<Vec<&'static str>> = LazyLock::new(|| {
|
||||
vec![
|
||||
@@ -646,4 +687,79 @@ mod tests {
|
||||
assert_eq!(metadata.get("cache-control"), Some(&"public".to_string()));
|
||||
assert!(!metadata.contains_key("authorization"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_metadata_from_mime_with_parquet_object_name() {
|
||||
let headers = HeaderMap::new();
|
||||
let mut metadata = HashMap::new();
|
||||
|
||||
extract_metadata_from_mime_with_object_name(&headers, &mut metadata, Some("data/test.parquet"));
|
||||
|
||||
assert_eq!(metadata.get("content-type"), Some(&"application/vnd.apache.parquet".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_metadata_from_mime_with_various_data_formats() {
|
||||
let test_cases = vec![
|
||||
("data.parquet", "application/vnd.apache.parquet"),
|
||||
("data.PARQUET", "application/vnd.apache.parquet"), // 测试大小写不敏感
|
||||
("file.avro", "application/avro"),
|
||||
("file.orc", "application/orc"),
|
||||
("file.feather", "application/feather"),
|
||||
("file.arrow", "application/arrow"),
|
||||
("file.json", "application/json"),
|
||||
("file.csv", "text/csv"),
|
||||
("file.txt", "text/plain"),
|
||||
("file.unknownext", "application/octet-stream"), // 使用真正未知的扩展名
|
||||
];
|
||||
|
||||
for (filename, expected_content_type) in test_cases {
|
||||
let headers = HeaderMap::new();
|
||||
let mut metadata = HashMap::new();
|
||||
|
||||
extract_metadata_from_mime_with_object_name(&headers, &mut metadata, Some(filename));
|
||||
|
||||
assert_eq!(
|
||||
metadata.get("content-type"),
|
||||
Some(&expected_content_type.to_string()),
|
||||
"Failed for filename: {filename}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_metadata_from_mime_with_existing_content_type() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("content-type", HeaderValue::from_static("custom/type"));
|
||||
|
||||
let mut metadata = HashMap::new();
|
||||
extract_metadata_from_mime_with_object_name(&headers, &mut metadata, Some("test.parquet"));
|
||||
|
||||
// 应该保留现有的 content-type,不被覆盖
|
||||
assert_eq!(metadata.get("content-type"), Some(&"custom/type".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_content_type_from_object_name() {
|
||||
// 测试 Parquet 文件(我们的自定义处理)
|
||||
assert_eq!(detect_content_type_from_object_name("test.parquet"), "application/vnd.apache.parquet");
|
||||
assert_eq!(detect_content_type_from_object_name("TEST.PARQUET"), "application/vnd.apache.parquet");
|
||||
|
||||
// 测试其他自定义数据格式
|
||||
assert_eq!(detect_content_type_from_object_name("data.avro"), "application/avro");
|
||||
assert_eq!(detect_content_type_from_object_name("data.orc"), "application/orc");
|
||||
assert_eq!(detect_content_type_from_object_name("data.feather"), "application/feather");
|
||||
assert_eq!(detect_content_type_from_object_name("data.arrow"), "application/arrow");
|
||||
|
||||
// 测试标准格式(mime_guess 处理)
|
||||
assert_eq!(detect_content_type_from_object_name("data.json"), "application/json");
|
||||
assert_eq!(detect_content_type_from_object_name("data.csv"), "text/csv");
|
||||
assert_eq!(detect_content_type_from_object_name("data.txt"), "text/plain");
|
||||
|
||||
// 测试真正未知的格式(使用一个 mime_guess 不认识的扩展名)
|
||||
assert_eq!(detect_content_type_from_object_name("unknown.unknownext"), "application/octet-stream");
|
||||
|
||||
// 测试没有扩展名的文件
|
||||
assert_eq!(detect_content_type_from_object_name("noextension"), "application/octet-stream");
|
||||
}
|
||||
}
|
||||
|
||||
+27
-27
@@ -7,21 +7,21 @@ type VersionParseResult = Result<(u32, u32, u32, Option<String>), Box<dyn std::e
|
||||
|
||||
#[allow(clippy::const_is_empty)]
|
||||
pub fn get_version() -> String {
|
||||
// 获取最新的 tag
|
||||
// Get the latest tag
|
||||
if let Ok(latest_tag) = get_latest_tag() {
|
||||
// 检查当前 commit 是否比最新 tag 更新
|
||||
// Check if current commit is newer than the latest tag
|
||||
if is_head_newer_than_tag(&latest_tag) {
|
||||
// 如果当前 commit 更新,则提升版本号
|
||||
// If current commit is newer, increment the version number
|
||||
if let Ok(new_version) = increment_version(&latest_tag) {
|
||||
return format!("refs/tags/{new_version}");
|
||||
}
|
||||
}
|
||||
|
||||
// 如果当前 commit 就是最新 tag,或者版本提升失败,返回当前 tag
|
||||
// If current commit is the latest tag, or version increment failed, return current tag
|
||||
return format!("refs/tags/{latest_tag}");
|
||||
}
|
||||
|
||||
// 如果没有 tag,使用原来的逻辑
|
||||
// If no tag exists, use original logic
|
||||
if !build::TAG.is_empty() {
|
||||
format!("refs/tags/{}", build::TAG)
|
||||
} else if !build::SHORT_COMMIT.is_empty() {
|
||||
@@ -31,7 +31,7 @@ pub fn get_version() -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取最新的 git tag
|
||||
/// Get the latest git tag
|
||||
fn get_latest_tag() -> Result<String, Box<dyn std::error::Error>> {
|
||||
let output = Command::new("git").args(["describe", "--tags", "--abbrev=0"]).output()?;
|
||||
|
||||
@@ -43,7 +43,7 @@ fn get_latest_tag() -> Result<String, Box<dyn std::error::Error>> {
|
||||
}
|
||||
}
|
||||
|
||||
/// 检查当前 HEAD 是否比指定的 tag 更新
|
||||
/// Check if current HEAD is newer than specified tag
|
||||
fn is_head_newer_than_tag(tag: &str) -> bool {
|
||||
let output = Command::new("git")
|
||||
.args(["merge-base", "--is-ancestor", tag, "HEAD"])
|
||||
@@ -55,23 +55,23 @@ fn is_head_newer_than_tag(tag: &str) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// 提升版本号(增加 patch 版本)
|
||||
/// Increment version number (increase patch version)
|
||||
fn increment_version(version: &str) -> Result<String, Box<dyn std::error::Error>> {
|
||||
// 解析版本号,例如 "1.0.0-alpha.19" -> (1, 0, 0, Some("alpha.19"))
|
||||
// Parse version number, e.g. "1.0.0-alpha.19" -> (1, 0, 0, Some("alpha.19"))
|
||||
let (major, minor, patch, pre_release) = parse_version(version)?;
|
||||
|
||||
// 如果有预发布标识符,则增加预发布版本号
|
||||
// If there's a pre-release identifier, increment the pre-release version number
|
||||
if let Some(pre) = pre_release {
|
||||
if let Some(new_pre) = increment_pre_release(&pre) {
|
||||
return Ok(format!("{major}.{minor}.{patch}-{new_pre}"));
|
||||
}
|
||||
}
|
||||
|
||||
// 否则增加 patch 版本号
|
||||
// Otherwise increment patch version number
|
||||
Ok(format!("{major}.{minor}.{}", patch + 1))
|
||||
}
|
||||
|
||||
/// 解析版本号
|
||||
/// Parse version number
|
||||
pub fn parse_version(version: &str) -> VersionParseResult {
|
||||
let parts: Vec<&str> = version.split('-').collect();
|
||||
let base_version = parts[0];
|
||||
@@ -89,9 +89,9 @@ pub fn parse_version(version: &str) -> VersionParseResult {
|
||||
Ok((major, minor, patch, pre_release))
|
||||
}
|
||||
|
||||
/// 增加预发布版本号
|
||||
/// Increment pre-release version number
|
||||
fn increment_pre_release(pre_release: &str) -> Option<String> {
|
||||
// 处理形如 "alpha.19" 的预发布版本
|
||||
// Handle pre-release versions like "alpha.19"
|
||||
let parts: Vec<&str> = pre_release.split('.').collect();
|
||||
if parts.len() == 2 {
|
||||
if let Ok(num) = parts[1].parse::<u32>() {
|
||||
@@ -99,7 +99,7 @@ fn increment_pre_release(pre_release: &str) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
// 处理形如 "alpha19" 的预发布版本
|
||||
// Handle pre-release versions like "alpha19"
|
||||
if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) {
|
||||
let prefix = &pre_release[..=pos];
|
||||
let suffix = &pre_release[pos + 1..];
|
||||
@@ -208,14 +208,14 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_parse_version() {
|
||||
// 测试标准版本解析
|
||||
// Test standard version parsing
|
||||
let (major, minor, patch, pre_release) = parse_version("1.0.0").unwrap();
|
||||
assert_eq!(major, 1);
|
||||
assert_eq!(minor, 0);
|
||||
assert_eq!(patch, 0);
|
||||
assert_eq!(pre_release, None);
|
||||
|
||||
// 测试预发布版本解析
|
||||
// Test pre-release version parsing
|
||||
let (major, minor, patch, pre_release) = parse_version("1.0.0-alpha.19").unwrap();
|
||||
assert_eq!(major, 1);
|
||||
assert_eq!(minor, 0);
|
||||
@@ -225,32 +225,32 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_increment_pre_release() {
|
||||
// 测试 alpha.19 -> alpha.20
|
||||
// Test alpha.19 -> alpha.20
|
||||
assert_eq!(increment_pre_release("alpha.19"), Some("alpha.20".to_string()));
|
||||
|
||||
// 测试 beta.5 -> beta.6
|
||||
// Test beta.5 -> beta.6
|
||||
assert_eq!(increment_pre_release("beta.5"), Some("beta.6".to_string()));
|
||||
|
||||
// 测试无法解析的情况
|
||||
// Test unparsable case
|
||||
assert_eq!(increment_pre_release("unknown"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_increment_version() {
|
||||
// 测试预发布版本递增
|
||||
// Test pre-release version increment
|
||||
assert_eq!(increment_version("1.0.0-alpha.19").unwrap(), "1.0.0-alpha.20");
|
||||
|
||||
// 测试标准版本递增
|
||||
// Test standard version increment
|
||||
assert_eq!(increment_version("1.0.0").unwrap(), "1.0.1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_version_format() {
|
||||
// 测试版本格式是否以 refs/tags/ 开头
|
||||
// Test if version format starts with refs/tags/
|
||||
let version = get_version();
|
||||
assert!(version.starts_with("refs/tags/") || version.starts_with("@"));
|
||||
|
||||
// 如果是 refs/tags/ 格式,应该包含版本号
|
||||
// If it's refs/tags/ format, should contain version number
|
||||
if let Some(version_part) = version.strip_prefix("refs/tags/") {
|
||||
assert!(!version_part.is_empty());
|
||||
}
|
||||
@@ -258,14 +258,14 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_current_version_output() {
|
||||
// 显示当前版本输出
|
||||
// Display current version output
|
||||
let version = get_version();
|
||||
println!("Current version: {version}");
|
||||
|
||||
// 验证版本格式
|
||||
// Verify version format
|
||||
assert!(version.starts_with("refs/tags/") || version.starts_with("@"));
|
||||
|
||||
// 如果是 refs/tags/ 格式,验证版本号不为空
|
||||
// If it's refs/tags/ format, verify version number is not empty
|
||||
if let Some(version_part) = version.strip_prefix("refs/tags/") {
|
||||
assert!(!version_part.is_empty());
|
||||
println!("Version part: {version_part}");
|
||||
|
||||
@@ -14,12 +14,12 @@
|
||||
|
||||
for i in {0..3}; do
|
||||
DIR="/data/rustfs$i"
|
||||
echo "处理 $DIR"
|
||||
echo "Processing $DIR"
|
||||
if [ -d "$DIR" ]; then
|
||||
echo "清空 $DIR"
|
||||
echo "Clearing $DIR"
|
||||
sudo rm -rf "$DIR"/* "$DIR"/.[!.]* "$DIR"/..?* 2>/dev/null || true
|
||||
echo "已清空 $DIR"
|
||||
echo "Cleared $DIR"
|
||||
else
|
||||
echo "$DIR 不存在,跳过"
|
||||
echo "$DIR does not exist, skipping"
|
||||
fi
|
||||
done
|
||||
+10
-10
@@ -14,32 +14,32 @@
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# 脚本名称:scp_to_servers.sh
|
||||
# Script name: scp_to_servers.sh
|
||||
|
||||
rm ./target/x86_64-unknown-linux-gnu/release/rustfs.zip
|
||||
# 压缩./target/x86_64-unknown-linux-gnu/release/rustfs
|
||||
# Compress ./target/x86_64-unknown-linux-gnu/release/rustfs
|
||||
zip -j ./target/x86_64-unknown-linux-gnu/release/rustfs.zip ./target/x86_64-unknown-linux-gnu/release/rustfs
|
||||
|
||||
# 上传到服务器
|
||||
# Upload to server
|
||||
LOCAL_FILE="./target/x86_64-unknown-linux-gnu/release/rustfs.zip"
|
||||
REMOTE_PATH="~"
|
||||
|
||||
# 必须传入IP参数,否则报错退出
|
||||
# IP parameter must be provided, otherwise exit with error
|
||||
if [ -z "$1" ]; then
|
||||
echo "用法: $0 <server_ip>"
|
||||
echo "请传入目标服务器IP地址"
|
||||
echo "Usage: $0 <server_ip>"
|
||||
echo "Please provide target server IP address"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SERVER_LIST=("root@$1")
|
||||
|
||||
# 遍历服务器列表
|
||||
# Iterate through server list
|
||||
for SERVER in "${SERVER_LIST[@]}"; do
|
||||
echo "正在将文件复制到服务器:$SERVER 目标路径:$REMOTE_PATH"
|
||||
echo "Copying file to server: $SERVER target path: $REMOTE_PATH"
|
||||
scp "$LOCAL_FILE" "${SERVER}:${REMOTE_PATH}"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "成功复制到 $SERVER"
|
||||
echo "Successfully copied to $SERVER"
|
||||
else
|
||||
echo "复制到 $SERVER 失败"
|
||||
echo "Failed to copy to $SERVER"
|
||||
fi
|
||||
done
|
||||
+44
-44
@@ -16,9 +16,9 @@
|
||||
|
||||
# ps -ef | grep rustfs | awk '{print $2}'| xargs kill -9
|
||||
|
||||
# 本地 rustfs.zip 路径
|
||||
# Local rustfs.zip path
|
||||
ZIP_FILE="./rustfs.zip"
|
||||
# 解压目标
|
||||
# Unzip target
|
||||
UNZIP_TARGET="./"
|
||||
|
||||
|
||||
@@ -35,119 +35,119 @@ SERVER_LIST=(
|
||||
|
||||
REMOTE_TMP="~/rustfs"
|
||||
|
||||
# 部署 rustfs 到所有服务器
|
||||
# Deploy rustfs to all servers
|
||||
deploy() {
|
||||
echo "解压 $ZIP_FILE ..."
|
||||
echo "Unzipping $ZIP_FILE ..."
|
||||
unzip -o "$ZIP_FILE" -d "$UNZIP_TARGET"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "解压失败,退出"
|
||||
echo "Unzip failed, exiting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LOCAL_RUSTFS="${UNZIP_TARGET}rustfs"
|
||||
if [ ! -f "$LOCAL_RUSTFS" ]; then
|
||||
echo "未找到解压后的 rustfs 文件,退出"
|
||||
echo "Unzipped rustfs file not found, exiting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for SERVER in "${SERVER_LIST[@]}"; do
|
||||
echo "上传 $LOCAL_RUSTFS 到 $SERVER:$REMOTE_TMP"
|
||||
echo "Uploading $LOCAL_RUSTFS to $SERVER:$REMOTE_TMP"
|
||||
scp "$LOCAL_RUSTFS" "${SERVER}:${REMOTE_TMP}"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "❌ 上传到 $SERVER 失败,跳过"
|
||||
echo "❌ Upload to $SERVER failed, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "在 $SERVER 上操作 systemctl 和文件替换"
|
||||
echo "Operating systemctl and file replacement on $SERVER"
|
||||
ssh "$SERVER" bash <<EOF
|
||||
set -e
|
||||
echo "停止 rustfs 服务"
|
||||
echo "Stopping rustfs service"
|
||||
sudo systemctl stop rustfs || true
|
||||
echo "覆盖 /usr/local/bin/rustfs"
|
||||
echo "Overwriting /usr/local/bin/rustfs"
|
||||
sudo cp ~/rustfs /usr/local/bin/rustfs
|
||||
sudo chmod +x /usr/local/bin/rustfs
|
||||
echo "启动 rustfs 服务"
|
||||
echo "Starting rustfs service"
|
||||
sudo systemctl start rustfs
|
||||
echo "检测 rustfs 服务状态"
|
||||
echo "Checking rustfs service status"
|
||||
sudo systemctl status rustfs --no-pager --lines=10
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ $SERVER 部署并重启 rustfs 成功"
|
||||
echo "✅ $SERVER deployed and restarted rustfs successfully"
|
||||
else
|
||||
echo "❌ $SERVER 部署或重启 rustfs 失败"
|
||||
echo "❌ $SERVER failed to deploy or restart rustfs"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# 清空 /data/rustfs0~3 目录下所有文件(包括隐藏文件)
|
||||
# Clear all files (including hidden files) in /data/rustfs0~3 directories
|
||||
clear_data_dirs() {
|
||||
for SERVER in "${SERVER_LIST[@]}"; do
|
||||
echo "清空 $SERVER:/data/rustfs0~3 下所有文件"
|
||||
echo "Clearing all files in $SERVER:/data/rustfs0~3"
|
||||
ssh "$SERVER" bash <<EOF
|
||||
for i in {0..3}; do
|
||||
DIR="/data/rustfs$i"
|
||||
echo "处理 $DIR"
|
||||
if [ -d "$DIR" ]; then
|
||||
echo "清空 $DIR"
|
||||
sudo rm -rf "$DIR"/* "$DIR"/.[!.]* "$DIR"/..?* 2>/dev/null || true
|
||||
echo "已清空 $DIR"
|
||||
DIR="/data/rustfs\$i"
|
||||
echo "Processing \$DIR"
|
||||
if [ -d "\$DIR" ]; then
|
||||
echo "Clearing \$DIR"
|
||||
sudo rm -rf "\$DIR"/* "\$DIR"/.[!.]* "\$DIR"/..?* 2>/dev/null || true
|
||||
echo "Cleared \$DIR"
|
||||
else
|
||||
echo "$DIR 不存在,跳过"
|
||||
echo "\$DIR does not exist, skipping"
|
||||
fi
|
||||
done
|
||||
EOF
|
||||
done
|
||||
}
|
||||
|
||||
# 控制 rustfs 服务
|
||||
# Control rustfs service
|
||||
stop_rustfs() {
|
||||
for SERVER in "${SERVER_LIST[@]}"; do
|
||||
echo "停止 $SERVER rustfs 服务"
|
||||
echo "Stopping $SERVER rustfs service"
|
||||
ssh "$SERVER" "sudo systemctl stop rustfs"
|
||||
done
|
||||
}
|
||||
|
||||
start_rustfs() {
|
||||
for SERVER in "${SERVER_LIST[@]}"; do
|
||||
echo "启动 $SERVER rustfs 服务"
|
||||
echo "Starting $SERVER rustfs service"
|
||||
ssh "$SERVER" "sudo systemctl start rustfs"
|
||||
done
|
||||
}
|
||||
|
||||
restart_rustfs() {
|
||||
for SERVER in "${SERVER_LIST[@]}"; do
|
||||
echo "重启 $SERVER rustfs 服务"
|
||||
echo "Restarting $SERVER rustfs service"
|
||||
ssh "$SERVER" "sudo systemctl restart rustfs"
|
||||
done
|
||||
}
|
||||
|
||||
# 向所有服务器追加公钥到 ~/.ssh/authorized_keys
|
||||
# Append public key to ~/.ssh/authorized_keys on all servers
|
||||
add_ssh_key() {
|
||||
if [ -z "$2" ]; then
|
||||
echo "用法: $0 addkey <pubkey_file>"
|
||||
echo "Usage: $0 addkey <pubkey_file>"
|
||||
exit 1
|
||||
fi
|
||||
PUBKEY_FILE="$2"
|
||||
if [ ! -f "$PUBKEY_FILE" ]; then
|
||||
echo "指定的公钥文件不存在: $PUBKEY_FILE"
|
||||
echo "Specified public key file does not exist: $PUBKEY_FILE"
|
||||
exit 1
|
||||
fi
|
||||
PUBKEY_CONTENT=$(cat "$PUBKEY_FILE")
|
||||
for SERVER in "${SERVER_LIST[@]}"; do
|
||||
echo "追加公钥到 $SERVER:~/.ssh/authorized_keys"
|
||||
echo "Appending public key to $SERVER:~/.ssh/authorized_keys"
|
||||
ssh "$SERVER" "mkdir -p ~/.ssh && chmod 700 ~/.ssh && echo '$PUBKEY_CONTENT' >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ $SERVER 公钥追加成功"
|
||||
echo "✅ $SERVER public key appended successfully"
|
||||
else
|
||||
echo "❌ $SERVER 公钥追加失败"
|
||||
echo "❌ $SERVER public key append failed"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
monitor_logs() {
|
||||
for SERVER in "${SERVER_LIST[@]}"; do
|
||||
echo "监控 $SERVER:/var/logs/rustfs/rustfs.log ..."
|
||||
echo "Monitoring $SERVER:/var/logs/rustfs/rustfs.log ..."
|
||||
ssh "$SERVER" "tail -F /var/logs/rustfs/rustfs.log" |
|
||||
sed "s/^/[$SERVER] /" &
|
||||
done
|
||||
@@ -156,32 +156,32 @@ monitor_logs() {
|
||||
|
||||
set_env_file() {
|
||||
if [ -z "$2" ]; then
|
||||
echo "用法: $0 setenv <env_file>"
|
||||
echo "Usage: $0 setenv <env_file>"
|
||||
exit 1
|
||||
fi
|
||||
ENV_FILE="$2"
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "指定的环境变量文件不存在: $ENV_FILE"
|
||||
echo "Specified environment variable file does not exist: $ENV_FILE"
|
||||
exit 1
|
||||
fi
|
||||
for SERVER in "${SERVER_LIST[@]}"; do
|
||||
echo "上传 $ENV_FILE 到 $SERVER:~/rustfs.env"
|
||||
echo "Uploading $ENV_FILE to $SERVER:~/rustfs.env"
|
||||
scp "$ENV_FILE" "${SERVER}:~/rustfs.env"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "❌ 上传到 $SERVER 失败,跳过"
|
||||
echo "❌ Upload to $SERVER failed, skipping"
|
||||
continue
|
||||
fi
|
||||
echo "覆盖 $SERVER:/etc/default/rustfs"
|
||||
echo "Overwriting $SERVER:/etc/default/rustfs"
|
||||
ssh "$SERVER" "sudo mv ~/rustfs.env /etc/default/rustfs"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ $SERVER /etc/default/rustfs 覆盖成功"
|
||||
echo "✅ $SERVER /etc/default/rustfs overwritten successfully"
|
||||
else
|
||||
echo "❌ $SERVER /etc/default/rustfs 覆盖失败"
|
||||
echo "❌ $SERVER /etc/default/rustfs overwrite failed"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# 主命令分发
|
||||
# Main command dispatcher
|
||||
case "$1" in
|
||||
deploy)
|
||||
deploy
|
||||
@@ -208,6 +208,6 @@ case "$1" in
|
||||
set_env_file "$@"
|
||||
;;
|
||||
*)
|
||||
echo "用法: $0 {deploy|clear|stop|start|restart|addkey <pubkey_file>|monitor_logs|setenv <env_file>}"
|
||||
echo "Usage: $0 {deploy|clear|stop|start|restart|addkey <pubkey_file>|monitor_logs|setenv <env_file>}"
|
||||
;;
|
||||
esac
|
||||
+1
-1
@@ -26,6 +26,6 @@ fi
|
||||
echo "Creating log directory if it does not exist..."
|
||||
mkdir -p "$current_dir/deploy/logs/notify"
|
||||
|
||||
# 启动 webhook 服务器
|
||||
# Start webhook server
|
||||
echo "Starting webhook server..."
|
||||
cargo run --example webhook -p rustfs-notify &
|
||||
+22
-22
@@ -47,19 +47,19 @@ export RUSTFS_ADDRESS=":9000"
|
||||
export RUSTFS_CONSOLE_ENABLE=true
|
||||
export RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# export RUSTFS_SERVER_DOMAINS="localhost:9000"
|
||||
# HTTPS 证书目录
|
||||
# HTTPS certificate directory
|
||||
# export RUSTFS_TLS_PATH="./deploy/certs"
|
||||
|
||||
# 可观测性 相关配置信息
|
||||
#export RUSTFS_OBS_ENDPOINT=http://localhost:4317 # OpenTelemetry Collector 的地址
|
||||
#export RUSTFS_OBS_USE_STDOUT=false # 是否使用标准输出
|
||||
#export RUSTFS_OBS_SAMPLE_RATIO=2.0 # 采样率,0.0-1.0之间,0.0表示不采样,1.0表示全部采样
|
||||
#export RUSTFS_OBS_METER_INTERVAL=1 # 采样间隔,单位为秒
|
||||
#export RUSTFS_OBS_SERVICE_NAME=rustfs # 服务名称
|
||||
#export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # 服务版本
|
||||
export RUSTFS_OBS_ENVIRONMENT=develop # 环境名称
|
||||
export RUSTFS_OBS_LOGGER_LEVEL=info # 日志级别,支持 trace, debug, info, warn, error
|
||||
export RUSTFS_OBS_LOCAL_LOGGING_ENABLED=true # 是否启用本地日志记录
|
||||
# Observability related configuration
|
||||
#export RUSTFS_OBS_ENDPOINT=http://localhost:4317 # OpenTelemetry Collector address
|
||||
#export RUSTFS_OBS_USE_STDOUT=false # Whether to use standard output
|
||||
#export RUSTFS_OBS_SAMPLE_RATIO=2.0 # Sample ratio, between 0.0-1.0, 0.0 means no sampling, 1.0 means full sampling
|
||||
#export RUSTFS_OBS_METER_INTERVAL=1 # Sampling interval in seconds
|
||||
#export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name
|
||||
#export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version
|
||||
export RUSTFS_OBS_ENVIRONMENT=develop # Environment name
|
||||
export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error
|
||||
export RUSTFS_OBS_LOCAL_LOGGING_ENABLED=true # Whether to enable local logging
|
||||
export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory
|
||||
export RUSTFS_OBS_LOG_ROTATION_TIME="hour" # Log rotation time unit, can be "second", "minute", "hour", "day"
|
||||
export RUSTFS_OBS_LOG_ROTATION_SIZE_MB=100 # Log rotation size in MB
|
||||
@@ -89,34 +89,34 @@ export OTEL_INSTRUMENTATION_SCHEMA_URL="https://opentelemetry.io/schemas/1.31.0"
|
||||
export OTEL_INSTRUMENTATION_ATTRIBUTES="env=production"
|
||||
|
||||
# notify
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENABLE="on" # 是否启用 webhook 通知
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT="http://[::]:3020/webhook" # webhook 通知地址
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENABLE="on" # Whether to enable webhook notification
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT="http://[::]:3020/webhook" # Webhook notification address
|
||||
export RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR="$current_dir/deploy/logs/notify"
|
||||
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY="on" # 是否启用 webhook 通知
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY="http://[::]:3020/webhook" # webhook 通知地址
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY="on" # Whether to enable webhook notification
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY="http://[::]:3020/webhook" # Webhook notification address
|
||||
export RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_PRIMARY="$current_dir/deploy/logs/notify"
|
||||
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENABLE_MASTER="on" # 是否启用 webhook 通知
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MASTER="http://[::]:3020/webhook" # webhook 通知地址
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENABLE_MASTER="on" # Whether to enable webhook notification
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MASTER="http://[::]:3020/webhook" # Webhook notification address
|
||||
export RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_MASTER="$current_dir/deploy/logs/notify"
|
||||
|
||||
|
||||
export RUSTFS_NS_SCANNER_INTERVAL=60 # 对象扫描间隔时间,单位为秒
|
||||
export RUSTFS_NS_SCANNER_INTERVAL=60 # Object scanning interval in seconds
|
||||
# exportRUSTFS_SKIP_BACKGROUND_TASK=true
|
||||
|
||||
export RUSTFS_COMPRESSION_ENABLED=true # 是否启用压缩
|
||||
export RUSTFS_COMPRESSION_ENABLED=true # Whether to enable compression
|
||||
|
||||
#export RUSTFS_REGION="us-east-1"
|
||||
|
||||
# 事件消息配置
|
||||
# Event message configuration
|
||||
#export RUSTFS_EVENT_CONFIG="./deploy/config/event.example.toml"
|
||||
|
||||
if [ -n "$1" ]; then
|
||||
export RUSTFS_VOLUMES="$1"
|
||||
fi
|
||||
|
||||
# 启动 webhook 服务器
|
||||
# Start webhook server
|
||||
#cargo run --example webhook -p rustfs-notify &
|
||||
# 启动主服务
|
||||
# Start main service
|
||||
cargo run --bin rustfs
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
echo "🔍 验证所有PR分支的CI状态..."
|
||||
|
||||
branches=(
|
||||
"feature/add-auth-module-tests"
|
||||
"feature/add-storage-core-tests"
|
||||
"feature/add-admin-handlers-tests"
|
||||
"feature/add-server-components-tests"
|
||||
"feature/add-integration-tests"
|
||||
)
|
||||
|
||||
cd /workspace
|
||||
|
||||
for branch in "${branches[@]}"; do
|
||||
echo ""
|
||||
echo "🌟 检查分支: $branch"
|
||||
|
||||
git checkout $branch 2>/dev/null
|
||||
|
||||
echo "📝 检查代码格式..."
|
||||
if cargo fmt --all --check; then
|
||||
echo "✅ 代码格式正确"
|
||||
else
|
||||
echo "❌ 代码格式有问题"
|
||||
fi
|
||||
|
||||
echo "🔧 检查基本编译..."
|
||||
if cargo check --quiet; then
|
||||
echo "✅ 基本编译通过"
|
||||
else
|
||||
echo "❌ 编译失败"
|
||||
fi
|
||||
|
||||
echo "🧪 运行核心测试..."
|
||||
if timeout 60 cargo test --lib --quiet 2>/dev/null; then
|
||||
echo "✅ 核心测试通过"
|
||||
else
|
||||
echo "⚠️ 测试超时或失败(可能是依赖问题)"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "🎉 所有分支检查完毕!"
|
||||
echo ""
|
||||
echo "📋 PR状态总结:"
|
||||
echo "- PR #309: feature/add-auth-module-tests"
|
||||
echo "- PR #313: feature/add-storage-core-tests"
|
||||
echo "- PR #314: feature/add-admin-handlers-tests"
|
||||
echo "- PR #315: feature/add-server-components-tests"
|
||||
echo "- PR #316: feature/add-integration-tests"
|
||||
echo ""
|
||||
echo "✅ 所有冲突已解决,代码已格式化"
|
||||
echo "🔗 请检查GitHub上的CI状态"
|
||||
Reference in New Issue
Block a user