mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-30 18:12:14 +00:00
Compare commits
105 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fa1554be7f | |||
| f08b592c6f | |||
| 8add0126f5 | |||
| 09a83a8f56 | |||
| a0f1bb4ff0 | |||
| 3ac1d2ab0b | |||
| 4aafb07173 | |||
| 8c76e9838b | |||
| d4dcb2ac9d | |||
| 41d2812861 | |||
| 960c13a34b | |||
| 989827e3b5 | |||
| a77be8f89b | |||
| 1525143a04 | |||
| b7a945e453 | |||
| 3796b684f0 | |||
| 1511f9eacb | |||
| 12f355a3bc | |||
| 83bac39417 | |||
| 177fe2ab44 | |||
| 1d1f00470d | |||
| 457f4e0170 | |||
| 93d0606cbd | |||
| ae7444ebb4 | |||
| b8c788ffca | |||
| 9677320f23 | |||
| 116db4f5d9 | |||
| f9b5ad17a9 | |||
| fb0d096d5d | |||
| a5de275875 | |||
| dc5ce7d0af | |||
| ac443a90ce | |||
| 2d0b3227c5 | |||
| 1cbf156559 | |||
| 03f8270a60 | |||
| 96fb06f48e | |||
| ffcf18f5f3 | |||
| 38eb0781cf | |||
| ce291ab610 | |||
| f77ccd5b23 | |||
| 8dc5ef6ef5 | |||
| f255b8a9f1 | |||
| 478720d2ee | |||
| 6b4172998b | |||
| 6ce24f3b63 | |||
| af93d2daba | |||
| 28edfd6190 | |||
| 579b124726 | |||
| 1ffe23e10f | |||
| 4615791193 | |||
| b2b92de26c | |||
| e05d07494e | |||
| c0d3f53f7a | |||
| 49366ee200 | |||
| 73e6542eea | |||
| f57dd5a3c7 | |||
| 1bda5ed636 | |||
| 27971f5c03 | |||
| 6506ae8c7b | |||
| 8223cda5ff | |||
| 8152c8e084 | |||
| da9dfb51e7 | |||
| c11efce1e4 | |||
| fa793237fb | |||
| df109680ae | |||
| 1eef75b0ca | |||
| 642d83f0e4 | |||
| 16b9189e9b | |||
| 1979fc7fb1 | |||
| 68d3dba9fc | |||
| db8ef63674 | |||
| 6d476ae9a1 | |||
| 2a3d127a86 | |||
| 8abf683178 | |||
| 5048ff8c69 | |||
| 711aab73c3 | |||
| 315e8134eb | |||
| 979626c370 | |||
| 505a566c7c | |||
| 8efa359a1a | |||
| 522c1675d8 | |||
| e160633ac4 | |||
| 458086dc80 | |||
| 0e697fa7e1 | |||
| 30cc481731 | |||
| b9f2369095 | |||
| 4ce374a181 | |||
| ff7dc77b5e | |||
| b387f6c58a | |||
| 64508ae770 | |||
| 6a2f98b468 | |||
| eb0dc24921 | |||
| c8c71e34b6 | |||
| 11552eb722 | |||
| b0b7f56281 | |||
| 120f1021b5 | |||
| 20e07519a1 | |||
| 5c4dadc0b7 | |||
| 0f7e7f35e9 | |||
| 5e97377aa0 | |||
| b49570d87a | |||
| d70ca1990e | |||
| 70fdf79297 | |||
| 4472324125 | |||
| b8c45fc9e3 |
@@ -2641,7 +2641,7 @@
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 73
|
||||
},
|
||||
@@ -2670,7 +2670,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rate(rustfs_system_process_network_io{job=~\"$job\", direction=\"received\"}[5m]))",
|
||||
"expr": "sum by (job) (rate(rustfs_system_network_host_network_io{job=~\"$job\", direction=\"received\"}[5m]))",
|
||||
"legendFormat": "RX - {{job}}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
@@ -2681,7 +2681,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rate(rustfs_system_process_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))",
|
||||
"expr": "sum by (job) (rate(rustfs_system_network_host_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))",
|
||||
"legendFormat": "TX - {{job}}",
|
||||
"range": true,
|
||||
"refId": "D"
|
||||
@@ -2690,6 +2690,115 @@
|
||||
"title": "Network",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "Bps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 73
|
||||
},
|
||||
"id": 500,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job, interface) (rate(rustfs_system_network_host_network_io_per_interface{job=~\"$job\", direction=\"received\"}[5m]))",
|
||||
"legendFormat": "RX {{interface}} - {{job}}",
|
||||
"range": true,
|
||||
"refId": "E"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job, interface) (rate(rustfs_system_network_host_network_io_per_interface{job=~\"$job\", direction=\"transmitted\"}[5m]))",
|
||||
"legendFormat": "TX {{interface}} - {{job}}",
|
||||
"range": true,
|
||||
"refId": "F"
|
||||
}
|
||||
],
|
||||
"title": "Network by Interface",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
|
||||
@@ -25,6 +25,12 @@ Current `test-and-lint` gate includes:
|
||||
|
||||
- `cargo nextest run --all --exclude e2e_test`
|
||||
- `cargo test --all --doc`
|
||||
- `cargo test -p rustfs get_object_chunk_fast_path`
|
||||
- `cargo test -p rustfs materialize_chunk_stream_before_commit`
|
||||
- `touch rustfs/build.rs`
|
||||
- `cargo build -p rustfs --bins --jobs 2`
|
||||
- `cargo test -p e2e_test archive_multipart_roundtrip_preserves_bytes`
|
||||
- `cargo test -p e2e_test presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path`
|
||||
- `cargo fmt --all --check`
|
||||
- `cargo clippy --all-targets --all-features -- -D warnings`
|
||||
- `./scripts/check_layer_dependencies.sh`
|
||||
|
||||
@@ -53,6 +53,8 @@ on:
|
||||
- ".github/workflows/docker.yml"
|
||||
- ".github/workflows/audit.yml"
|
||||
- ".github/workflows/performance.yml"
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
schedule:
|
||||
- cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -17,6 +17,8 @@ name: CLA Check
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
issue_comment:
|
||||
types: [created, edited]
|
||||
|
||||
@@ -31,7 +33,26 @@ jobs:
|
||||
if: ${{ github.event_name != 'issue_comment' || github.event.issue.pull_request }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Report CLA result for merge queue
|
||||
if: github.event_name == 'merge_group'
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
await github.rest.checks.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: 'CLA Check',
|
||||
head_sha: context.sha,
|
||||
status: 'completed',
|
||||
conclusion: 'success',
|
||||
output: {
|
||||
title: 'CLA requirements satisfied for merge queue',
|
||||
summary: 'Queued pull requests must satisfy the required CLA check before they enter the merge queue. This reports the existing CLA result on the merge-group SHA.'
|
||||
}
|
||||
});
|
||||
|
||||
- name: Create token for rustfs/cla
|
||||
if: github.event_name != 'merge_group'
|
||||
id: registry-token
|
||||
uses: actions/create-github-app-token@v3
|
||||
with:
|
||||
@@ -42,6 +63,7 @@ jobs:
|
||||
permission-contents: write
|
||||
|
||||
- name: Run CLA Bot
|
||||
if: github.event_name != 'merge_group'
|
||||
uses: overtrue/cla-bot@v0.0.9
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
|
||||
@@ -6,24 +6,39 @@ Use the nearest subdirectory `AGENTS.md` for path-specific guidance.
|
||||
## Rule Precedence
|
||||
|
||||
1. System/developer instructions.
|
||||
2. This file (global defaults).
|
||||
3. The nearest `AGENTS.md` in the current path (more specific scope wins).
|
||||
2. Current user/task instructions.
|
||||
3. The nearest `AGENTS.md` in the current path.
|
||||
4. This file (global defaults).
|
||||
|
||||
If repo-level instructions conflict, follow the nearest file and keep behavior aligned with CI.
|
||||
|
||||
## Execution Discipline
|
||||
|
||||
- Read the relevant existing code, tests, and local guidance before changing behavior.
|
||||
- State assumptions when they affect the implementation or verification path.
|
||||
- If a task has multiple plausible interpretations, list the options briefly and choose the narrowest reasonable path; ask when the ambiguity would make the change risky.
|
||||
- For multi-step work, keep the plan minimal and tied to verifiable outcomes.
|
||||
- Avoid redundant file reads, repeated commands, and unnecessary exploratory work once enough context is available.
|
||||
- A good result is a minimal diff with clear assumptions, no over-engineering, and independent verification.
|
||||
|
||||
## Communication and Language
|
||||
|
||||
- Respond in the same language used by the requester.
|
||||
- Keep source code, comments, commit messages, and PR title/body in English.
|
||||
- Be concise. Avoid sycophantic openers, closing fluff, and verbose status reporting.
|
||||
|
||||
## Change Style for Existing Logic
|
||||
|
||||
- Prefer direct, local code over extracting one-off helpers.
|
||||
- Extract a helper only when logic is reused or the extraction materially clarifies a non-trivial flow.
|
||||
- Solve only the requested problem; do not add speculative features, configurability, or adjacent improvements.
|
||||
- Prefer editing existing code over rewriting files or reshaping unrelated logic.
|
||||
- Modify only what is required and remove only artifacts introduced by your own changes.
|
||||
- Preserve the existing control-flow and logic shape when fixing bugs or addressing review comments, especially in init, distributed coordination, locking, metadata, and concurrency paths.
|
||||
- Do not refactor existing code only to make it easier to unit test.
|
||||
- Keep fixes narrowly aligned with the requested behavior; avoid semantic-adjacent rewrites while touching sensitive paths.
|
||||
- Keep code elegant, concise, and direct. Prefer minimal, readable implementations over over-engineering and excessive abstraction. Use comments to clarify non-obvious intent and invariants, not to compensate for unclear code.
|
||||
- Mention unrelated issues when useful, but do not fix them as part of a narrow task.
|
||||
|
||||
## Constant and String Usage
|
||||
|
||||
@@ -44,6 +59,8 @@ Reference the source files above instead.
|
||||
|
||||
## Verification Before PR
|
||||
|
||||
Convert changes into independently verifiable outcomes. Prefer focused tests for behavior changes and run the relevant checks before declaring completion.
|
||||
|
||||
For code changes, run and pass the following before opening a PR:
|
||||
|
||||
```bash
|
||||
@@ -69,6 +86,7 @@ Do not open a PR with code changes when the required checks fail.
|
||||
- Use `N/A` for non-applicable template sections.
|
||||
- Include verification commands in the PR description.
|
||||
- When using `gh pr create`/`gh pr edit`, use `--body-file` instead of inline `--body` for multiline markdown.
|
||||
- Do not include the literal sequence `\n` in any GitHub issue, pull request, or discussion comment.
|
||||
- After fixing code review comments or CI findings, always mark corresponding review
|
||||
comments/threads as resolved before returning to the user.
|
||||
- In handling review comments, confirm the underlying issue before changing code.
|
||||
|
||||
+376
@@ -0,0 +1,376 @@
|
||||
# ARCHITECTURE.md
|
||||
|
||||
> Last updated: 2026-04-13 · Revision: 1 (draft)
|
||||
>
|
||||
> This document describes the high-level architecture of RustFS.
|
||||
> If you want to familiarize yourself with the code base, you are in the right place!
|
||||
>
|
||||
> See also [CONTRIBUTING.md](CONTRIBUTING.md) for development workflow.
|
||||
|
||||
## Bird's Eye View
|
||||
|
||||
RustFS is a high-performance, S3-compatible distributed object storage system written
|
||||
in Rust. It uses erasure coding for data durability, supports multi-tenancy through
|
||||
IAM/STS, and provides a web-based admin console.
|
||||
|
||||
A running RustFS node exposes:
|
||||
|
||||
- **S3 API** (port 9000) — the primary data path for object CRUD
|
||||
- **Admin API** (port 9000, `/minio/` prefix) — cluster management, IAM, metrics
|
||||
- **Console** (port 9001) — web UI backed by the Admin API
|
||||
- **Inter-node RPC** (gRPC/tonic) — cluster communication for distributed mode
|
||||
|
||||
The core data flow for a PUT request looks like:
|
||||
|
||||
```
|
||||
HTTP request
|
||||
→ server (TLS, auth, routing, compression)
|
||||
→ app/object_usecase (validation, policy, lifecycle)
|
||||
→ storage/ecfs (erasure coding, encryption, checksums)
|
||||
→ ecstore (disk pool selection, data distribution)
|
||||
→ rio (reader pipeline: encrypt → compress → hash → write)
|
||||
→ io-core (zero-copy I/O, buffer pool, direct I/O)
|
||||
→ local disk / remote disk via RPC
|
||||
```
|
||||
|
||||
## Code Map
|
||||
|
||||
The repository is a Cargo workspace with a flat `crates/` layout:
|
||||
|
||||
```
|
||||
rustfs/ # Workspace root (virtual manifest)
|
||||
├── rustfs/ # Main binary + library crate (75K lines)
|
||||
│ └── src/
|
||||
│ ├── main.rs # Entry point, startup sequence
|
||||
│ ├── lib.rs # Module tree root
|
||||
│ ├── server/ # HTTP server, TLS, routing, middleware
|
||||
│ ├── admin/ # Admin API handlers and console
|
||||
│ ├── app/ # Use-case layer (object, bucket, multipart)
|
||||
│ ├── storage/ # Storage engine interface and implementation
|
||||
│ ├── auth.rs # S3 request authentication
|
||||
│ ├── config/ # CLI args, config parsing, workload profiles
|
||||
│ └── ...
|
||||
├── crates/ # 39 library crates
|
||||
│ ├── ecstore/ # Erasure-coded storage engine (⚠️ 87K lines)
|
||||
│ ├── rio/ # Reader I/O pipeline (encrypt, compress, hash)
|
||||
│ ├── io-core/ # Zero-copy I/O, scheduling, buffer pool
|
||||
│ ├── io-metrics/ # I/O metrics collection
|
||||
│ ├── common/ # Shared runtime state, globals, data usage types
|
||||
│ ├── config/ # Configuration types and parsing
|
||||
│ ├── utils/ # Pure utility functions
|
||||
│ ├── ... # (see "Crate Reference" below)
|
||||
│ └── e2e_test/ # End-to-end integration tests
|
||||
└── docs/ # Design documents and analysis
|
||||
```
|
||||
|
||||
### Main Crate Layers (`rustfs/src/`)
|
||||
|
||||
The main crate is organized in layers, top to bottom:
|
||||
|
||||
| Layer | Directory | Responsibility |
|
||||
|-------|-----------|----------------|
|
||||
| **Server** | `server/` | HTTP listener, TLS, CORS, compression, middleware, graceful shutdown |
|
||||
| **Admin** | `admin/` | Admin API routing, 30+ handler modules, web console |
|
||||
| **App** | `app/` | Use-case orchestration: object_usecase, bucket_usecase, multipart_usecase |
|
||||
| **Storage** | `storage/` | S3 API translation, erasure-coded FS, SSE encryption, RPC, concurrency |
|
||||
| **Auth** | `auth.rs` | S3 signature verification, credential validation |
|
||||
| **Config** | `config/` | CLI parsing, config struct, workload profiles |
|
||||
|
||||
A request flows **downward** through the layers. No layer should reach upward
|
||||
(e.g., storage must not import from admin).
|
||||
|
||||
### Crate Reference
|
||||
|
||||
Crates are organized in a dependency DAG with 9 depth levels (0 = leaf, 8 = top):
|
||||
|
||||
```
|
||||
Depth 0 — LEAF (no internal deps):
|
||||
appauth, checksums, config, credentials, crypto, io-metrics,
|
||||
madmin, s3-common, workers, zip
|
||||
|
||||
Depth 1:
|
||||
io-core (→ io-metrics)
|
||||
policy (→ config, credentials, crypto)
|
||||
utils (→ config) ⚠️ inverted: utils should be leaf
|
||||
|
||||
Depth 2:
|
||||
concurrency, filemeta, keystone, kms, lock, obs,
|
||||
signer, targets, trusted-proxies
|
||||
|
||||
Depth 3:
|
||||
common (→ filemeta, madmin) ⚠️ inverted: common should be leaf
|
||||
|
||||
Depth 4:
|
||||
object-capacity, protos, rio
|
||||
|
||||
Depth 5 — CORE:
|
||||
ecstore (16 internal deps, 11 dependents — the architectural heart)
|
||||
|
||||
Depth 6:
|
||||
audit, heal, iam, metrics, notify, s3select-api, scanner
|
||||
|
||||
Depth 7:
|
||||
object-io, protocols, s3select-query
|
||||
|
||||
Depth 8 — TOP:
|
||||
rustfs (35 internal deps — the binary, depends on almost everything)
|
||||
```
|
||||
|
||||
#### By Domain
|
||||
|
||||
**Core Infrastructure:**
|
||||
|
||||
| Crate | Lines | Purpose |
|
||||
|-------|-------|---------|
|
||||
| `config` | 3.3K | Configuration types and environment parsing |
|
||||
| `utils` | 8.7K | Pure utilities (paths, compression, network, retry) |
|
||||
| `common` | 4.4K | Shared runtime state, globals, data usage types, metrics |
|
||||
| `madmin` | 5.5K | Admin API request/response types |
|
||||
|
||||
**I/O Pipeline:**
|
||||
|
||||
| Crate | Lines | Purpose |
|
||||
|-------|-------|---------|
|
||||
| `io-core` | 6.5K | Zero-copy I/O, buffer pool, direct I/O, scheduling, backpressure |
|
||||
| `io-metrics` | 4.5K | I/O operation metrics and counters |
|
||||
| `rio` | 6.9K | Composable reader chain (encrypt → compress → hash → limit) |
|
||||
| `object-io` | 2.4K | High-level object read/write using rio + ecstore |
|
||||
| `concurrency` | 1.8K | Concurrency control wrappers over io-core |
|
||||
|
||||
**Storage Engine:**
|
||||
|
||||
| Crate | Lines | Purpose |
|
||||
|-------|-------|---------|
|
||||
| `ecstore` | 87K | ⚠️ Erasure-coded storage: disks, pools, buckets, replication, lifecycle |
|
||||
| `filemeta` | 10K | File/object metadata types and versioning |
|
||||
| `checksums` | 732 | Checksum computation |
|
||||
| `lock` | 7.1K | Distributed lock manager |
|
||||
| `heal` | 5.9K | Data healing / bitrot repair |
|
||||
| `scanner` | 5.4K | Background data usage scanner |
|
||||
| `object-capacity` | 2.5K | Capacity tracking and management |
|
||||
|
||||
**Security & Auth:**
|
||||
|
||||
| Crate | Lines | Purpose |
|
||||
|-------|-------|---------|
|
||||
| `crypto` | 1.6K | Encryption primitives |
|
||||
| `credentials` | 713 | Credential types (access key / secret key) |
|
||||
| `signer` | 1.4K | S3 v4 request signing |
|
||||
| `iam` | 9.0K | Identity and access management |
|
||||
| `policy` | 8.8K | Policy engine (S3 bucket/IAM policies) |
|
||||
| `kms` | 8.1K | Key management service integration |
|
||||
| `keystone` | 1.9K | OpenStack Keystone auth |
|
||||
| `appauth` | 143 | Application-level auth tokens |
|
||||
|
||||
**Protocol & API:**
|
||||
|
||||
| Crate | Lines | Purpose |
|
||||
|-------|-------|---------|
|
||||
| `protos` | 5.7K | Protobuf/gRPC definitions for inter-node RPC |
|
||||
| `protocols` | 18K | FTP/FTPS, WebDAV, Swift API support |
|
||||
| `s3-common` | 738 | Shared S3 types |
|
||||
| `s3select-api` | 1.9K | S3 Select interface |
|
||||
| `s3select-query` | 3.6K | S3 Select query engine |
|
||||
|
||||
**Observability:**
|
||||
|
||||
| Crate | Lines | Purpose |
|
||||
|-------|-------|---------|
|
||||
| `metrics` | 8.4K | Prometheus metric collectors |
|
||||
| `io-metrics` | 4.5K | I/O-specific metrics |
|
||||
| `obs` | 5.6K | OpenTelemetry tracing and telemetry |
|
||||
| `audit` | 2.4K | Audit logging |
|
||||
|
||||
**Events:**
|
||||
|
||||
| Crate | Lines | Purpose |
|
||||
|-------|-------|---------|
|
||||
| `notify` | 5.5K | Event notification system |
|
||||
| `targets` | 3.2K | Notification targets (Kafka, AMQP, webhook, etc.) |
|
||||
|
||||
**Other:**
|
||||
|
||||
| Crate | Lines | Purpose |
|
||||
|-------|-------|---------|
|
||||
| `trusted-proxies` | 4.0K | Trusted proxy / IP forwarding |
|
||||
| `zip` | 986 | ZIP archive support for bulk downloads |
|
||||
| `workers` | 136 | Simple worker abstraction |
|
||||
|
||||
## Architecture Invariants
|
||||
|
||||
> These are rules that the codebase should follow. Some are currently violated
|
||||
> (marked with ⚠️). Documenting them here makes the violations explicit and
|
||||
> trackable.
|
||||
|
||||
1. **Layers flow downward.** Server → Admin/App → Storage → ecstore → rio/io-core.
|
||||
No upward imports.
|
||||
|
||||
2. **Leaf crates have zero internal dependencies.** `config`, `credentials`, `crypto`,
|
||||
`io-metrics`, `madmin`, `s3-common` should depend only on external crates.
|
||||
- ⚠️ VIOLATED: `utils` depends on `config`, `common` depends on `filemeta` and `madmin`.
|
||||
|
||||
3. **Each type has exactly one definition.** Types shared across crates must be defined
|
||||
in one crate and re-exported or imported by others.
|
||||
- ⚠️ VIOLATED: `ReplicationStats` (4 copies), `LastMinuteLatency` (3 copies),
|
||||
`BackpressureConfig` (3 copies), `DataUsageInfo` (2 copies).
|
||||
|
||||
4. **ecstore does not know about HTTP or S3 protocol details.** It operates on
|
||||
storage-level abstractions (objects, buckets, disks, pools).
|
||||
|
||||
5. **The `rustfs` binary crate is the only place that wires everything together.**
|
||||
Individual crates should be testable in isolation.
|
||||
|
||||
6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
|
||||
not bare `Error`).
|
||||
- ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`;
|
||||
`heal` use `anyhow` in library code.
|
||||
|
||||
## Known Structural Issues
|
||||
|
||||
> This section documents known problems in the current architecture.
|
||||
> It exists so the team can track and address them deliberately.
|
||||
|
||||
### Critical
|
||||
|
||||
- **common/scanner code duplication (~3K lines).** `scanner` depends on `common`
|
||||
but maintains its own copies of `DataUsageInfo`, `LastMinuteLatency`, and related
|
||||
types instead of importing them.
|
||||
|
||||
- **ecstore is a monolith (87K lines, 163 files).** It contains disk management,
|
||||
bucket management, erasure coding, replication, lifecycle, RPC, and configuration
|
||||
— all in one crate. It should be decomposed along its existing subdirectories.
|
||||
|
||||
### High
|
||||
|
||||
- **Dependency inversions.** `utils → config` and `common → filemeta/madmin` break
|
||||
the layering model. These need to be untangled.
|
||||
|
||||
- **Three-layer BackpressureConfig/DeadlockConfig duplication** across io-core,
|
||||
concurrency, and rustfs/storage. Should be defined once with builder/composition.
|
||||
|
||||
### Medium
|
||||
|
||||
- **Inconsistent error handling.** Three strategies (thiserror/snafu/anyhow) and
|
||||
mixed naming (bare `Error` vs descriptive names).
|
||||
|
||||
- **Ambiguous common vs utils boundary.** Both described as "utilities and data
|
||||
structures." Need clear ownership rules.
|
||||
|
||||
## Cross-Cutting Concerns
|
||||
|
||||
### Error Handling
|
||||
|
||||
The project convention is `thiserror` for typed errors with descriptive names.
|
||||
See `AGENTS.md`: "Prefer thiserror for library-facing error types."
|
||||
|
||||
```rust
|
||||
// GOOD
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum StorageError {
|
||||
#[error("disk not found: {0}")]
|
||||
DiskNotFound(String),
|
||||
}
|
||||
|
||||
// AVOID
|
||||
pub enum Error { ... } // too generic
|
||||
anyhow::Result<T> // in library code (OK in tests/CLI)
|
||||
```
|
||||
|
||||
### Logging & Tracing
|
||||
|
||||
- Use `tracing` crate (`info!`, `warn!`, `error!`, `debug!`, `trace!`)
|
||||
- Structured fields: `tracing::info!(bucket = %name, "created bucket")`
|
||||
- Spans for request-scoped context
|
||||
|
||||
### Metrics
|
||||
|
||||
- Prometheus-style metrics via `rustfs-obs` runtime and schema
|
||||
- I/O-specific counters via `rustfs-io-metrics`
|
||||
- Registration happens at crate level, collection/reporting in `rustfs-obs`
|
||||
|
||||
### Testing
|
||||
|
||||
- Unit tests: `#[cfg(test)] mod tests` in the same file
|
||||
- Integration tests: inside respective crates (not top-level `tests/`)
|
||||
- E2E tests: `crates/e2e_test/` — tests against a running server
|
||||
- Run all: `make test` or `cargo nextest run`
|
||||
|
||||
## Startup Sequence
|
||||
|
||||
The binary (`main.rs`) boots in this order:
|
||||
|
||||
1. Environment variable compatibility (`MINIO_*` → `RUSTFS_*`)
|
||||
2. Tokio runtime construction
|
||||
3. CLI argument parsing
|
||||
4. License, observability, TLS, trusted proxies initialization
|
||||
5. Config parsing, server address resolution
|
||||
6. Credentials, endpoints, local disks, lock client initialization
|
||||
7. Capacity management initialization
|
||||
8. HTTP server start (S3 API + optional console)
|
||||
9. ECStore initialization (erasure coding storage engine)
|
||||
10. Global config, background replication, KMS
|
||||
11. Optional: FTP/FTPS/WebDAV servers
|
||||
12. Event notifier, audit system, deadlock detector
|
||||
13. Bucket metadata, IAM, Keystone, OIDC
|
||||
14. Scanner and heal manager
|
||||
15. Metrics system, mark `FullReady`
|
||||
16. Wait for shutdown signal → graceful shutdown
|
||||
|
||||
## Dependency Diagram (Simplified)
|
||||
|
||||
```
|
||||
┌─────────┐
|
||||
│ rustfs │ (binary + lib, 75K lines)
|
||||
│ main │
|
||||
└────┬────┘
|
||||
│
|
||||
┌───────────────┼───────────────┐
|
||||
│ │ │
|
||||
┌────▼────┐ ┌────▼────┐ ┌─────▼─────┐
|
||||
│ server │ │ admin │ │ app │
|
||||
│ (HTTP) │ │(console)│ │(use-cases) │
|
||||
└────┬────┘ └────┬────┘ └─────┬─────┘
|
||||
│ │ │
|
||||
└───────────────┼───────────────┘
|
||||
│
|
||||
┌──────▼──────┐
|
||||
│ storage │
|
||||
│ (ecfs, SSE, │
|
||||
│ RPC, ACL) │
|
||||
└──────┬──────┘
|
||||
│
|
||||
┌──────────────────┼──────────────────┐
|
||||
│ │ │
|
||||
┌─────▼─────┐ ┌──────▼──────┐ ┌──────▼──────┐
|
||||
│ ecstore │ │ rio │ │ io-core │
|
||||
│ (87K,core) │ │ (readers) │ │ (zero-copy) │
|
||||
└─────┬──────┘ └─────────────┘ └─────────────┘
|
||||
│
|
||||
┌─────┬──┼──┬─────┬──────┐
|
||||
│ │ │ │ │ │
|
||||
common utils config policy filemeta ...
|
||||
```
|
||||
|
||||
## How to Navigate
|
||||
|
||||
- **"Where does S3 PutObject go?"**
|
||||
`server/` routes → `app/object_usecase` validates → `storage/ecfs` encodes →
|
||||
`ecstore` distributes → `rio` encrypts/compresses → `io-core` writes
|
||||
|
||||
- **"Where are bucket policies enforced?"**
|
||||
`app/bucket_usecase` calls into `crates/policy/`
|
||||
|
||||
- **"Where is replication configured?"**
|
||||
`admin/handlers/replication.rs` and `admin/handlers/site_replication.rs` for API,
|
||||
`ecstore/src/bucket/replication/` for engine
|
||||
|
||||
- **"Where do I add a new admin endpoint?"**
|
||||
Add handler in `admin/handlers/`, register in `admin/router.rs`
|
||||
|
||||
- **"Where do I add a new metric?"**
|
||||
Define descriptor/collector in `crates/obs/src/metrics/`, expose via `/minio/v2/metrics`
|
||||
|
||||
---
|
||||
|
||||
*Inspired by [matklad's ARCHITECTURE.md](https://matklad.github.io/2021/02/06/ARCHITECTURE.md.html)
|
||||
and [rust-analyzer's architecture.md](https://github.com/rust-analyzer/rust-analyzer/blob/master/docs/book/src/contributing/architecture.md).*
|
||||
Generated
+669
-466
File diff suppressed because it is too large
Load Diff
+25
-29
@@ -31,8 +31,6 @@ members = [
|
||||
"crates/kms", # Key Management Service
|
||||
"crates/lock", # Distributed locking implementation
|
||||
"crates/madmin", # Management dashboard and admin API interface
|
||||
"crates/mcp", # MCP server for S3 operations
|
||||
"crates/metrics", # Metrics collection and reporting
|
||||
"crates/notify", # Notification system for events
|
||||
"crates/obs", # Observability utilities
|
||||
"crates/object-capacity", # Capacity scan and refresh core
|
||||
@@ -40,7 +38,7 @@ members = [
|
||||
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
|
||||
"crates/protos", # Protocol buffer definitions
|
||||
"crates/rio", # Rust I/O utilities and abstractions
|
||||
"crates/concurrency", # Rust I/O utilities and abstractions
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-common", # Common utilities and data structures for S3 compatibility
|
||||
"crates/s3select-api", # S3 Select API interface
|
||||
"crates/s3select-query", # S3 Select query engine
|
||||
@@ -52,7 +50,6 @@ members = [
|
||||
"crates/workers", # Worker thread pools and task scheduling
|
||||
"crates/io-metrics", # Zero-copy metrics collection for performance analysis
|
||||
"crates/io-core", # Zero-copy core reader and writer implementations
|
||||
"crates/object-io", # Object I/O policy and zero-copy helper primitives
|
||||
"crates/zip", # ZIP file handling and compression
|
||||
]
|
||||
resolver = "3"
|
||||
@@ -74,6 +71,8 @@ unsafe_code = "deny"
|
||||
|
||||
[workspace.lints.clippy]
|
||||
all = "warn"
|
||||
needless_collect = "warn"
|
||||
redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
@@ -94,16 +93,14 @@ rustfs-keystone = { path = "crates/keystone", version = "0.0.5" }
|
||||
rustfs-kms = { path = "crates/kms", version = "0.0.5" }
|
||||
rustfs-lock = { path = "crates/lock", version = "0.0.5" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
|
||||
rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
|
||||
rustfs-metrics = { path = "crates/metrics", version = "0.0.5" }
|
||||
rustfs-notify = { path = "crates/notify", version = "0.0.5" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "0.0.5" }
|
||||
rustfs-object-io = { path = "crates/object-io", version = "0.0.5" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" }
|
||||
rustfs-obs = { path = "crates/obs", version = "0.0.5" }
|
||||
rustfs-policy = { path = "crates/policy", version = "0.0.5" }
|
||||
rustfs-protos = { path = "crates/protos", version = "0.0.5" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
|
||||
rustfs-rio = { path = "crates/rio", version = "0.0.5" }
|
||||
rustfs-s3-common = { path = "crates/s3-common", version = "0.0.5" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
|
||||
@@ -115,27 +112,28 @@ rustfs-targets = { path = "crates/targets", version = "0.0.5" }
|
||||
rustfs-utils = { path = "crates/utils", version = "0.0.5" }
|
||||
rustfs-workers = { path = "crates/workers", version = "0.0.5" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async-compression = { version = "0.4.41" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.89"
|
||||
axum = "0.8.8"
|
||||
async-nats = "0.47.0"
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.32"
|
||||
futures-core = "0.3.32"
|
||||
futures-util = "0.3.32"
|
||||
pollster = "0.4.0"
|
||||
pulsar = { version = "6.7.2", default-features = false, features = ["tokio-rustls-runtime"] }
|
||||
hyper = { version = "1.9.0", features = ["http2", "http1", "server"] }
|
||||
hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
|
||||
hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
|
||||
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
http = "1.4.0"
|
||||
http-body = "1.0.1"
|
||||
http-body-util = "0.1.3"
|
||||
reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
|
||||
socket2 = { version = "0.6.3", features = ["all"] }
|
||||
tokio = { version = "1.51.1", features = ["fs", "rt-multi-thread"] }
|
||||
tokio = { version = "1.52.1", features = ["fs", "rt-multi-thread"] }
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
|
||||
tokio-stream = { version = "0.1.18" }
|
||||
tokio-test = "0.4.5"
|
||||
@@ -154,18 +152,16 @@ flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.2"
|
||||
prost = "0.14.3"
|
||||
quick-xml = "0.39.2"
|
||||
rmcp = { version = "1.3.0" }
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.149", features = ["raw_value"] }
|
||||
serde_urlencoded = "0.7.1"
|
||||
schemars = "1.2.1"
|
||||
|
||||
# Cryptography and Security
|
||||
aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
|
||||
argon2 = { version = "0.6.0-rc.8" }
|
||||
blake2 = "0.11.0-rc.5"
|
||||
blake2 = "0.11.0-rc.6"
|
||||
chacha20poly1305 = { version = "0.11.0-rc.3" }
|
||||
crc-fast = "1.9.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
@@ -173,7 +169,7 @@ jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
|
||||
openidconnect = { version = "4.0", default-features = false }
|
||||
pbkdf2 = "0.13.0-rc.10"
|
||||
rsa = { version = "0.10.0-rc.17" }
|
||||
rustls = { version = "0.23.37", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls = { version = "0.23.38", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-pki-types = "1.14.0"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.0"
|
||||
@@ -202,7 +198,7 @@ base64 = "0.22.1"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.2"
|
||||
cfg-if = "1.0.4"
|
||||
clap = { version = "4.6.0", features = ["derive", "env"] }
|
||||
clap = { version = "4.6.1", features = ["derive", "env"] }
|
||||
const-str = { version = "1.1.0", features = ["std", "proc"] }
|
||||
convert_case = "0.11.0"
|
||||
criterion = { version = "0.8", features = ["html_reports"] }
|
||||
@@ -210,21 +206,21 @@ crossbeam-queue = "0.3.12"
|
||||
crossbeam-channel = "0.5.15"
|
||||
crossbeam-deque = "0.8.6"
|
||||
crossbeam-utils = "0.8.21"
|
||||
datafusion = "53.0.0"
|
||||
datafusion = "53.1.0"
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.10"
|
||||
faster-hex = "0.10.0"
|
||||
flate2 = "1.1.9"
|
||||
glob = "0.3.3"
|
||||
google-cloud-storage = "1.10.0"
|
||||
google-cloud-auth = "1.8.0"
|
||||
hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
|
||||
google-cloud-storage = "1.11.0"
|
||||
google-cloud-auth = "1.9.0"
|
||||
hashbrown = { version = "0.17.0", features = ["serde", "rayon"] }
|
||||
hex = "0.4.3"
|
||||
hex-simd = "0.8.0"
|
||||
highway = { version = "1.3.0" }
|
||||
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
||||
lazy_static = "1.5.0"
|
||||
libc = "0.2.184"
|
||||
libc = "0.2.185"
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.11"
|
||||
memmap2 = "0.9.10"
|
||||
@@ -244,17 +240,17 @@ path-clean = "1.0.1"
|
||||
percent-encoding = "2.3.2"
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.0", features = ["serde"] }
|
||||
rand = { version = "0.10.1", features = ["serde"] }
|
||||
ratelimit = "0.10.1"
|
||||
rayon = "1.11.0"
|
||||
rayon = "1.12.0"
|
||||
reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.12.3" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.29.0", features = ["websocket"] }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.30.0", features = ["websocket"] }
|
||||
rustix = { version = "1.1.4", features = ["fs"] }
|
||||
rust-embed = { version = "8.11.0" }
|
||||
rustc-hash = { version = "2.1.2" }
|
||||
s3s = { git = "https://github.com/rustfs/s3s", rev = "010ae03bdd8c93a9c046108190987bb0e6cb0692", features = ["minio"] }
|
||||
s3s = { git = "https://github.com/rustfs/s3s", rev = "a3b16608df35aaeed8fff08b4988d03f4ca9445b", features = ["minio"] }
|
||||
serial_test = "3.4.0"
|
||||
shadow-rs = { version = "1.7.1", default-features = false }
|
||||
siphasher = "1.0.2"
|
||||
@@ -270,14 +266,14 @@ tempfile = "3.27.0"
|
||||
test-case = "3.3.1"
|
||||
thiserror = "2.0.18"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.4"
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = "0.32.1"
|
||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.23.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
uuid = { version = "1.23.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
walkdir = "2.5.0"
|
||||
wildmatch = { version = "2.6.1", features = ["serde"] }
|
||||
@@ -288,7 +284,7 @@ zstd = "0.13.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.3"
|
||||
dial9-tokio-telemetry = { version = "0.2", git = "https://github.com/dial9-rs/dial9-tokio-telemetry.git", rev = "60502082601b647c4a51962595721f631b7bbce1" }
|
||||
dial9-tokio-telemetry = "0.3"
|
||||
opentelemetry = { version = "0.31.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
|
||||
opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] }
|
||||
@@ -320,7 +316,7 @@ jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
|
||||
pprof = { package = "pprof-pyroscope-fork", version = "0.1500.3", features = ["flamegraph", "protobuf-codec"] }
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
ignored = ["rustfs", "rustfs-mcp"]
|
||||
ignored = ["rustfs"]
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
|
||||
+3
-3
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM alpine:3.23 AS build
|
||||
FROM alpine:3.23.4 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
@@ -54,7 +54,7 @@ RUN set -eux; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
|
||||
FROM alpine:3.23
|
||||
FROM alpine:3.23.4
|
||||
|
||||
ARG RELEASE=latest
|
||||
ARG BUILD_DATE
|
||||
@@ -73,7 +73,7 @@ LABEL name="RustFS" \
|
||||
license="Apache-2.0"
|
||||
|
||||
RUN apk update && \
|
||||
apk add --no-cache ca-certificates coreutils curl "zlib>=1.3.2-r0"
|
||||
apk add --no-cache ca-certificates coreutils curl
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
|
||||
@@ -100,28 +100,28 @@ To get started with RustFS, follow these steps:
|
||||
curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
|
||||
```
|
||||
|
||||
### 2\. Docker Quick Start (Option 2)
|
||||
### 2. Docker Quick Start (Option 2)
|
||||
|
||||
The RustFS container runs as a non-root user `rustfs` (UID `10001`). If you run Docker with `-v` to mount a host directory, please ensure the host directory owner is set to `10001`, otherwise you will encounter permission denied errors.
|
||||
|
||||
```bash
|
||||
# Create data and logs directories
|
||||
mkdir -p data logs
|
||||
# Create data and logs directories
|
||||
mkdir -p data logs
|
||||
|
||||
# Change the owner of these directories
|
||||
chown -R 10001:10001 data logs
|
||||
# Change the owner of these directories
|
||||
chown -R 10001:10001 data logs
|
||||
|
||||
# Using latest version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
# Using latest version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-alpha.76
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-alpha.76
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
```bash
|
||||
podman run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
podman run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
```
|
||||
|
||||
You can also use Docker Compose. Using the `docker-compose.yml` file in the root directory:
|
||||
@@ -219,6 +219,48 @@ rustfs --help
|
||||
|
||||
**NOTE**: To access the RustFS instance via `https`, please refer to the [TLS Configuration Docs](https://docs.rustfs.com/integration/tls-configured.html).
|
||||
|
||||
### OIDC Roles Claim (Microsoft Entra ID)
|
||||
|
||||
RustFS supports mapping an OIDC claim containing role values into the existing
|
||||
authorization pipeline. The `roles_claim` setting is **optional**: when unset or
|
||||
empty, only the `groups` claim contributes to authorization (same as older
|
||||
RustFS releases). For Microsoft Entra ID app roles, set `roles_claim=roles` so
|
||||
both console admin checks and bucket IAM policies can evaluate those roles.
|
||||
|
||||
Example environment configuration (opt-in roles claim):
|
||||
|
||||
```bash
|
||||
RUSTFS_IDENTITY_OPENID_ENABLE=on
|
||||
RUSTFS_IDENTITY_OPENID_CONFIG_URL="https://login.microsoftonline.com/<tenant-id>/v2.0/.well-known/openid-configuration"
|
||||
RUSTFS_IDENTITY_OPENID_CLIENT_ID="<client-id>"
|
||||
RUSTFS_IDENTITY_OPENID_CLIENT_SECRET="<client-secret>"
|
||||
RUSTFS_IDENTITY_OPENID_SCOPES="openid,profile,email"
|
||||
RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM="groups"
|
||||
RUSTFS_IDENTITY_OPENID_ROLES_CLAIM="roles"
|
||||
```
|
||||
|
||||
Policy condition example (evaluate app roles directly with `jwt:roles`; when
|
||||
`roles_claim` is configured, RustFS also merges those values into `jwt:groups`
|
||||
for backward compatibility with older policies):
|
||||
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["admin:*"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
"Condition": {
|
||||
"ForAnyValue:StringEquals": {
|
||||
"jwt:roles": ["RustFS.ConsoleAdmin"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
For detailed documentation, including configuration options, API references, and advanced usage, please visit our [Documentation](https://docs.rustfs.com).
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@ abd = "abd"
|
||||
mak = "mak"
|
||||
gae = "gae"
|
||||
GAE = "GAE"
|
||||
writeable = "writeable"
|
||||
thr = "thr"
|
||||
# s3-tests original test names (cannot be changed)
|
||||
nonexisted = "nonexisted"
|
||||
consts = "consts"
|
||||
|
||||
+48
-172
@@ -14,28 +14,19 @@
|
||||
|
||||
use crate::AuditEntry;
|
||||
use async_trait::async_trait;
|
||||
use hashbrown::HashSet;
|
||||
use rumqttc::QoS;
|
||||
use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
|
||||
use rustfs_config::{
|
||||
AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
|
||||
MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
|
||||
MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
|
||||
WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
|
||||
WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
|
||||
};
|
||||
use rustfs_config::AUDIT_DEFAULT_DIR;
|
||||
use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
|
||||
use rustfs_ecstore::config::KVS;
|
||||
use rustfs_targets::{
|
||||
Target,
|
||||
error::TargetError,
|
||||
target::{
|
||||
mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
|
||||
webhook::WebhookArgs,
|
||||
config::{
|
||||
build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
|
||||
validate_pulsar_config, validate_webhook_config,
|
||||
},
|
||||
error::TargetError,
|
||||
target::TargetType,
|
||||
};
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, warn};
|
||||
use url::Url;
|
||||
use std::collections::HashSet;
|
||||
|
||||
/// Trait for creating targets from configuration
|
||||
#[async_trait]
|
||||
@@ -49,10 +40,6 @@ pub trait TargetFactory: Send + Sync {
|
||||
/// Returns a set of valid configuration field names for this target type.
|
||||
/// This is used to filter environment variables.
|
||||
fn get_valid_fields(&self) -> HashSet<String>;
|
||||
|
||||
/// Returns a set of valid configuration env field names for this target type.
|
||||
/// This is used to filter environment variables.
|
||||
fn get_valid_env_fields(&self) -> HashSet<String>;
|
||||
}
|
||||
|
||||
/// Factory for creating Webhook targets
|
||||
@@ -61,71 +48,18 @@ pub struct WebhookTargetFactory;
|
||||
#[async_trait]
|
||||
impl TargetFactory for WebhookTargetFactory {
|
||||
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
|
||||
// All config values are now read directly from the merged `config` KVS.
|
||||
let endpoint = config
|
||||
.lookup(WEBHOOK_ENDPOINT)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
|
||||
let parsed_endpoint = endpoint.trim();
|
||||
let endpoint_url = Url::parse(parsed_endpoint)
|
||||
.map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
|
||||
|
||||
let args = WebhookArgs {
|
||||
enable: true, // If we are here, it's already enabled.
|
||||
endpoint: endpoint_url,
|
||||
auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
|
||||
queue_dir: config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
|
||||
queue_limit: config
|
||||
.lookup(WEBHOOK_QUEUE_LIMIT)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(DEFAULT_LIMIT),
|
||||
client_cert: config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(),
|
||||
client_key: config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(),
|
||||
client_ca: config.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
|
||||
skip_tls_verify: config
|
||||
.lookup(WEBHOOK_SKIP_TLS_VERIFY)
|
||||
.and_then(|v| v.parse::<bool>().ok())
|
||||
.unwrap_or(RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT),
|
||||
target_type: rustfs_targets::target::TargetType::AuditLog,
|
||||
};
|
||||
|
||||
let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
|
||||
let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?;
|
||||
Ok(Box::new(target))
|
||||
}
|
||||
|
||||
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
|
||||
// Validation also uses the merged `config` KVS directly.
|
||||
let endpoint = config
|
||||
.lookup(WEBHOOK_ENDPOINT)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
|
||||
debug!("endpoint: {}", endpoint);
|
||||
let parsed_endpoint = endpoint.trim();
|
||||
Url::parse(parsed_endpoint)
|
||||
.map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
|
||||
|
||||
let client_cert = config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default();
|
||||
let client_key = config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default();
|
||||
|
||||
if client_cert.is_empty() != client_key.is_empty() {
|
||||
return Err(TargetError::Configuration(
|
||||
"Both client_cert and client_key must be specified together".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let queue_dir = config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string());
|
||||
if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
|
||||
return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
validate_webhook_config(config, AUDIT_DEFAULT_DIR)
|
||||
}
|
||||
|
||||
fn get_valid_fields(&self) -> HashSet<String> {
|
||||
AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
fn get_valid_env_fields(&self) -> HashSet<String> {
|
||||
ENV_AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Factory for creating MQTT targets
|
||||
@@ -134,112 +68,54 @@ pub struct MQTTTargetFactory;
|
||||
#[async_trait]
|
||||
impl TargetFactory for MQTTTargetFactory {
|
||||
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
|
||||
let broker = config
|
||||
.lookup(MQTT_BROKER)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
|
||||
let broker_url = Url::parse(&broker)
|
||||
.map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
|
||||
|
||||
let topic = config
|
||||
.lookup(MQTT_TOPIC)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing MQTT topic".to_string()))?;
|
||||
|
||||
let args = MQTTArgs {
|
||||
enable: true, // Assumed enabled.
|
||||
broker: broker_url,
|
||||
topic,
|
||||
qos: config
|
||||
.lookup(MQTT_QOS)
|
||||
.and_then(|v| v.parse::<u8>().ok())
|
||||
.map(|q| match q {
|
||||
0 => QoS::AtMostOnce,
|
||||
1 => QoS::AtLeastOnce,
|
||||
2 => QoS::ExactlyOnce,
|
||||
_ => QoS::AtLeastOnce,
|
||||
})
|
||||
.unwrap_or(QoS::AtLeastOnce),
|
||||
username: config.lookup(MQTT_USERNAME).unwrap_or_default(),
|
||||
password: config.lookup(MQTT_PASSWORD).unwrap_or_default(),
|
||||
max_reconnect_interval: config
|
||||
.lookup(MQTT_RECONNECT_INTERVAL)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(Duration::from_secs)
|
||||
.unwrap_or_else(|| Duration::from_secs(5)),
|
||||
keep_alive: config
|
||||
.lookup(MQTT_KEEP_ALIVE_INTERVAL)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(Duration::from_secs)
|
||||
.unwrap_or_else(|| Duration::from_secs(30)),
|
||||
tls: MQTTTlsConfig::from_values(
|
||||
config.lookup(MQTT_TLS_POLICY).as_deref(),
|
||||
config.lookup(MQTT_TLS_CA).as_deref(),
|
||||
config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
|
||||
config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
|
||||
config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
|
||||
config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
|
||||
)?,
|
||||
queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
|
||||
queue_limit: config
|
||||
.lookup(MQTT_QUEUE_LIMIT)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(DEFAULT_LIMIT),
|
||||
target_type: rustfs_targets::target::TargetType::AuditLog,
|
||||
};
|
||||
|
||||
let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
|
||||
let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?;
|
||||
Ok(Box::new(target))
|
||||
}
|
||||
|
||||
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
|
||||
let broker = config
|
||||
.lookup(MQTT_BROKER)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
|
||||
let url = Url::parse(&broker)
|
||||
.map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
|
||||
|
||||
let tls = MQTTTlsConfig::from_values(
|
||||
config.lookup(MQTT_TLS_POLICY).as_deref(),
|
||||
config.lookup(MQTT_TLS_CA).as_deref(),
|
||||
config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
|
||||
config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
|
||||
config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
|
||||
config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
|
||||
)?;
|
||||
validate_mqtt_broker_url(&url, &tls)?;
|
||||
|
||||
if config.lookup(MQTT_TOPIC).is_none() {
|
||||
return Err(TargetError::Configuration("Missing MQTT topic".to_string()));
|
||||
}
|
||||
|
||||
if let Some(qos_str) = config.lookup(MQTT_QOS) {
|
||||
let qos = qos_str
|
||||
.parse::<u8>()
|
||||
.map_err(|_| TargetError::Configuration("Invalid QoS value".to_string()))?;
|
||||
if qos > 2 {
|
||||
return Err(TargetError::Configuration("QoS must be 0, 1, or 2".to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
let queue_dir = config.lookup(MQTT_QUEUE_DIR).unwrap_or_default();
|
||||
if !queue_dir.is_empty() {
|
||||
if !std::path::Path::new(&queue_dir).is_absolute() {
|
||||
return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
|
||||
}
|
||||
if let Some(qos_str) = config.lookup(MQTT_QOS)
|
||||
&& qos_str == "0"
|
||||
{
|
||||
warn!("Using queue_dir with QoS 0 may result in event loss");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
validate_mqtt_config(config)
|
||||
}
|
||||
|
||||
fn get_valid_fields(&self) -> HashSet<String> {
|
||||
AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn get_valid_env_fields(&self) -> HashSet<String> {
|
||||
ENV_AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
|
||||
pub struct NATSTargetFactory;
|
||||
|
||||
#[async_trait]
|
||||
impl TargetFactory for NATSTargetFactory {
|
||||
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
|
||||
let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
|
||||
let target = rustfs_targets::target::nats::NATSTarget::new(id, args)?;
|
||||
Ok(Box::new(target))
|
||||
}
|
||||
|
||||
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
|
||||
validate_nats_config(config, AUDIT_DEFAULT_DIR)
|
||||
}
|
||||
|
||||
fn get_valid_fields(&self) -> HashSet<String> {
|
||||
AUDIT_NATS_KEYS.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct PulsarTargetFactory;
|
||||
|
||||
#[async_trait]
|
||||
impl TargetFactory for PulsarTargetFactory {
|
||||
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
|
||||
let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
|
||||
let target = rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?;
|
||||
Ok(Box::new(target))
|
||||
}
|
||||
|
||||
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
|
||||
validate_pulsar_config(config, AUDIT_DEFAULT_DIR)
|
||||
}
|
||||
|
||||
fn get_valid_fields(&self) -> HashSet<String> {
|
||||
AUDIT_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{AuditEntry, AuditResult, AuditSystem};
|
||||
use crate::{AuditEntry, AuditResult, AuditSystem, system::AuditTargetMetricSnapshot};
|
||||
use rustfs_ecstore::config::Config;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tracing::{debug, error, trace, warn};
|
||||
@@ -89,6 +89,15 @@ pub async fn reload_audit_config(config: Config) -> AuditResult<()> {
|
||||
with_audit_system!(|system: Arc<AuditSystem>| async move { system.reload_config(config).await })
|
||||
}
|
||||
|
||||
/// Returns per-target audit delivery metrics for Prometheus collection.
|
||||
pub async fn audit_target_metrics() -> Vec<AuditTargetMetricSnapshot> {
|
||||
if let Some(system) = audit_system() {
|
||||
system.snapshot_target_metrics().await
|
||||
} else {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the global audit system is running
|
||||
pub async fn is_audit_system_running() -> bool {
|
||||
if let Some(system) = audit_system() {
|
||||
|
||||
@@ -31,4 +31,4 @@ pub use error::{AuditError, AuditResult};
|
||||
pub use global::*;
|
||||
pub use observability::{AuditMetrics, AuditMetricsReport, PerformanceValidation};
|
||||
pub use registry::AuditRegistry;
|
||||
pub use system::AuditSystem;
|
||||
pub use system::{AuditSystem, AuditTargetMetricSnapshot};
|
||||
|
||||
+15
-135
@@ -14,18 +14,17 @@
|
||||
|
||||
use crate::{
|
||||
AuditEntry, AuditError, AuditResult,
|
||||
factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory},
|
||||
factory::{MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory},
|
||||
};
|
||||
use futures::StreamExt;
|
||||
use futures::stream::FuturesUnordered;
|
||||
use hashbrown::{HashMap, HashSet};
|
||||
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, audit::AUDIT_ROUTE_PREFIX};
|
||||
use hashbrown::HashMap;
|
||||
use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
|
||||
use rustfs_ecstore::config::{Config, KVS};
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use rustfs_targets::{Target, TargetError, target::ChannelTargetType};
|
||||
use std::str::FromStr;
|
||||
use rustfs_targets::{Target, TargetError, config::collect_target_configs, target::ChannelTargetType};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing::{error, info};
|
||||
|
||||
/// Registry for managing audit targets
|
||||
pub struct AuditRegistry {
|
||||
@@ -52,6 +51,8 @@ impl AuditRegistry {
|
||||
// Register built-in factories
|
||||
registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory));
|
||||
registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
|
||||
registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
|
||||
registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
|
||||
|
||||
registry
|
||||
}
|
||||
@@ -105,143 +106,22 @@ impl AuditRegistry {
|
||||
&self,
|
||||
config: &Config,
|
||||
) -> AuditResult<Vec<Box<dyn Target<AuditEntry> + Send + Sync>>> {
|
||||
// Collect only environment variables with the relevant prefix to reduce memory usage
|
||||
let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
|
||||
// A collection of asynchronous tasks for concurrently executing target creation
|
||||
let mut tasks = FuturesUnordered::new();
|
||||
// 1. Traverse all registered plants and process them by target type
|
||||
for (target_type, factory) in &self.factories {
|
||||
tracing::Span::current().record("target_type", target_type.as_str());
|
||||
info!("Start working on target types...");
|
||||
|
||||
// 2. Prepare the configuration source
|
||||
// 2.1. Get the configuration segment in the file, e.g. 'audit_webhook'
|
||||
let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
|
||||
let file_configs = config.0.get(§ion_name).cloned().unwrap_or_default();
|
||||
// 2.2. Get the default configuration for that type
|
||||
let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
|
||||
debug!(?default_cfg, "Get the default configuration");
|
||||
|
||||
// *** Optimization point 1: Get all legitimate fields of the current target type ***
|
||||
let valid_fields = factory.get_valid_fields();
|
||||
debug!(?valid_fields, "Get the legitimate configuration fields");
|
||||
|
||||
// 3. Resolve instance IDs and configuration overrides from environment variables
|
||||
let mut instance_ids_from_env = HashSet::new();
|
||||
// 3.1. Instance discovery: Based on the '..._ENABLE_INSTANCEID' format
|
||||
let enable_prefix =
|
||||
format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
|
||||
.to_uppercase();
|
||||
for (key, value) in &all_env {
|
||||
if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
|
||||
&& let Some(id) = key.strip_prefix(&enable_prefix)
|
||||
&& !id.is_empty()
|
||||
{
|
||||
instance_ids_from_env.insert(id.to_lowercase());
|
||||
}
|
||||
}
|
||||
|
||||
// 3.2. Parse all relevant environment variable configurations
|
||||
// 3.2.1. Build environment variable prefixes such as 'RUSTFS_AUDIT_WEBHOOK_'
|
||||
let env_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}").to_uppercase();
|
||||
// 3.2.2. 'env_overrides' is used to store configurations parsed from environment variables in the format: {instance id -> {field -> value}}
|
||||
let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();
|
||||
for (key, value) in &all_env {
|
||||
if let Some(rest) = key.strip_prefix(&env_prefix) {
|
||||
// Use rsplitn to split from the right side to properly extract the INSTANCE_ID at the end
|
||||
// Format: <FIELD_NAME>_<INSTANCE_ID> or <FIELD_NAME>
|
||||
let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
|
||||
|
||||
// The first part from the right is INSTANCE_ID
|
||||
let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
|
||||
// The remaining part is FIELD_NAME
|
||||
let field_name_part = parts.next();
|
||||
|
||||
let (field_name, instance_id) = match field_name_part {
|
||||
// Case 1: The format is <FIELD_NAME>_<INSTANCE_ID>
|
||||
// e.g., rest = "ENDPOINT_PRIMARY" -> field_name="ENDPOINT", instance_id="PRIMARY"
|
||||
Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()),
|
||||
// Case 2: The format is <FIELD_NAME> (without INSTANCE_ID)
|
||||
// e.g., rest = "ENABLE" -> field_name="ENABLE", instance_id="" (Universal configuration `_ DEFAULT_DELIMITER`)
|
||||
None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()),
|
||||
};
|
||||
|
||||
// *** Optimization point 2: Verify whether the parsed field_name is legal ***
|
||||
if !field_name.is_empty() && valid_fields.contains(&field_name) {
|
||||
debug!(
|
||||
instance_id = %if instance_id.is_empty() { DEFAULT_DELIMITER } else { &instance_id },
|
||||
%field_name,
|
||||
%value,
|
||||
"Parsing to environment variables"
|
||||
);
|
||||
env_overrides
|
||||
.entry(instance_id)
|
||||
.or_default()
|
||||
.insert(field_name, value.clone());
|
||||
} else {
|
||||
// Ignore illegal field names
|
||||
warn!(
|
||||
field_name = %field_name,
|
||||
"Ignore environment variable fields, not found in the list of valid fields for target type {}",
|
||||
target_type
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
debug!(?env_overrides, "Complete the environment variable analysis");
|
||||
|
||||
// 4. Determine all instance IDs that need to be processed
|
||||
let mut all_instance_ids: HashSet<String> =
|
||||
file_configs.keys().filter(|k| *k != DEFAULT_DELIMITER).cloned().collect();
|
||||
all_instance_ids.extend(instance_ids_from_env);
|
||||
debug!(?all_instance_ids, "Determine all instance IDs");
|
||||
|
||||
// 5. Merge configurations and create tasks for each instance
|
||||
for id in all_instance_ids {
|
||||
// 5.1. Merge configuration, priority: Environment variables > File instance configuration > File default configuration
|
||||
let mut merged_config = default_cfg.clone();
|
||||
// Instance-specific configuration in application files
|
||||
if let Some(file_instance_cfg) = file_configs.get(&id) {
|
||||
merged_config.extend(file_instance_cfg.clone());
|
||||
}
|
||||
// Application instance-specific environment variable configuration
|
||||
if let Some(env_instance_cfg) = env_overrides.get(&id) {
|
||||
// Convert HashMap<String, String> to KVS
|
||||
let mut kvs_from_env = KVS::new();
|
||||
for (k, v) in env_instance_cfg {
|
||||
kvs_from_env.insert(k.clone(), v.clone());
|
||||
}
|
||||
merged_config.extend(kvs_from_env);
|
||||
}
|
||||
debug!(instance_id = %id, ?merged_config, "Complete configuration merge");
|
||||
|
||||
// 5.2. Check if the instance is enabled
|
||||
let enabled = merged_config
|
||||
.lookup(ENABLE_KEY)
|
||||
.map(|v| {
|
||||
EnableState::from_str(v.as_str())
|
||||
.ok()
|
||||
.map(|s| s.is_enabled())
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false);
|
||||
|
||||
if enabled {
|
||||
info!(instance_id = %id, "Target is enabled, ready to create a task");
|
||||
// 5.3. Create asynchronous tasks for enabled instances
|
||||
let tid = id.clone();
|
||||
let merged_config_arc = Arc::new(merged_config);
|
||||
tasks.push(async move {
|
||||
let result = factory.create_target(tid.clone(), &merged_config_arc).await;
|
||||
(tid, result)
|
||||
});
|
||||
} else {
|
||||
info!(instance_id = %id, "Skip disabled target");
|
||||
}
|
||||
for (id, merged_config) in collect_target_configs(config, AUDIT_ROUTE_PREFIX, target_type, &valid_fields) {
|
||||
info!(instance_id = %id, "Target is enabled, ready to create a task");
|
||||
let tid = id.clone();
|
||||
let merged_config_arc = Arc::new(merged_config);
|
||||
tasks.push(async move {
|
||||
let result = factory.create_target(tid.clone(), &merged_config_arc).await;
|
||||
(tid, result)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Concurrently execute all creation tasks and collect results
|
||||
let mut successful_targets = Vec::new();
|
||||
while let Some((id, result)) = tasks.next().await {
|
||||
match result {
|
||||
|
||||
@@ -24,6 +24,14 @@ use std::sync::Arc;
|
||||
use tokio::sync::{Mutex, RwLock, mpsc};
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AuditTargetMetricSnapshot {
|
||||
pub failed_messages: u64,
|
||||
pub queue_length: u64,
|
||||
pub target_id: String,
|
||||
pub total_messages: u64,
|
||||
}
|
||||
|
||||
/// State of the audit system
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AuditSystemState {
|
||||
@@ -537,8 +545,14 @@ impl AuditSystem {
|
||||
TargetError::Timeout(_) => {
|
||||
warn!("Timeout sending to target {}, retrying...", target.id());
|
||||
}
|
||||
TargetError::Dropped(reason) => {
|
||||
warn!("Dropped queued payload for target {}: {}", target.id(), reason);
|
||||
observability::record_target_failure();
|
||||
break;
|
||||
}
|
||||
_ => {
|
||||
error!("Permanent error for target {}: {}", target.id(), e);
|
||||
target.record_final_failure();
|
||||
observability::record_target_failure();
|
||||
break;
|
||||
}
|
||||
@@ -552,6 +566,7 @@ impl AuditSystem {
|
||||
|
||||
if retries >= MAX_RETRIES && !success {
|
||||
warn!("Max retries exceeded for key {}, target: {}, skipping", key.to_string(), target.id());
|
||||
target.record_final_failure();
|
||||
observability::record_target_failure();
|
||||
}
|
||||
}
|
||||
@@ -664,6 +679,25 @@ impl AuditSystem {
|
||||
registry.list_target_values()
|
||||
}
|
||||
|
||||
/// Returns per-target delivery metrics for Prometheus collection.
|
||||
pub async fn snapshot_target_metrics(&self) -> Vec<AuditTargetMetricSnapshot> {
|
||||
let targets = self.get_target_values().await;
|
||||
let mut snapshots = Vec::with_capacity(targets.len());
|
||||
|
||||
for target in targets {
|
||||
let delivery = target.delivery_snapshot();
|
||||
snapshots.push(AuditTargetMetricSnapshot {
|
||||
failed_messages: delivery.failed_messages,
|
||||
queue_length: delivery.queue_length,
|
||||
target_id: target.id().to_string(),
|
||||
total_messages: delivery.total_messages,
|
||||
});
|
||||
}
|
||||
|
||||
snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
|
||||
snapshots
|
||||
}
|
||||
|
||||
/// Gets information about a specific target
|
||||
///
|
||||
/// # Arguments
|
||||
|
||||
@@ -51,14 +51,14 @@ impl AllTierStats {
|
||||
pub fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
|
||||
for (tier, st) in tiers {
|
||||
self.tiers
|
||||
.insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st));
|
||||
.insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: AllTierStats) {
|
||||
for (tier, st) in other.tiers {
|
||||
self.tiers
|
||||
.insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st));
|
||||
.insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -673,7 +673,7 @@ impl DataUsageCache {
|
||||
let mut leaves = Vec::new();
|
||||
let mut remove = total - limit;
|
||||
add(self, path, &mut leaves);
|
||||
leaves.sort_by(|a, b| a.objects.cmp(&b.objects));
|
||||
leaves.sort_by_key(|a| a.objects);
|
||||
|
||||
while remove > 0 && !leaves.is_empty() {
|
||||
let Some(e) = leaves.first() else {
|
||||
@@ -1097,7 +1097,7 @@ impl DataUsageInfo {
|
||||
|
||||
/// Add bucket usage info
|
||||
pub fn add_bucket_usage(&mut self, bucket: String, usage: BucketUsageInfo) {
|
||||
self.buckets_usage.insert(bucket.clone(), usage);
|
||||
self.buckets_usage.insert(bucket, usage);
|
||||
self.buckets_count = self.buckets_usage.len() as u64;
|
||||
self.last_update = Some(SystemTime::now());
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ use std::{
|
||||
fmt::{self, Display},
|
||||
sync::OnceLock,
|
||||
};
|
||||
use tokio::sync::{broadcast, mpsc};
|
||||
use tokio::sync::{broadcast, mpsc, oneshot};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const HEAL_DELETE_DANGLING: bool = true;
|
||||
@@ -206,11 +206,59 @@ pub struct HealOpts {
|
||||
pub set: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealAdmissionDropReason {
|
||||
QueueFull,
|
||||
PolicyDropped,
|
||||
}
|
||||
|
||||
impl HealAdmissionDropReason {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::QueueFull => "queue_full",
|
||||
Self::PolicyDropped => "policy_dropped",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealAdmissionResult {
|
||||
Accepted,
|
||||
Merged,
|
||||
Full,
|
||||
Dropped(HealAdmissionDropReason),
|
||||
}
|
||||
|
||||
impl HealAdmissionResult {
|
||||
pub fn result_label(self) -> &'static str {
|
||||
match self {
|
||||
Self::Accepted => "accepted",
|
||||
Self::Merged => "merged",
|
||||
Self::Full => "full",
|
||||
Self::Dropped(_) => "dropped",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reason_label(self) -> &'static str {
|
||||
match self {
|
||||
Self::Dropped(reason) => reason.as_str(),
|
||||
_ => "none",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_admitted(self) -> bool {
|
||||
matches!(self, Self::Accepted | Self::Merged)
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal channel command type
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug)]
|
||||
pub enum HealChannelCommand {
|
||||
/// Start a new heal task
|
||||
Start(HealChannelRequest),
|
||||
Start {
|
||||
request: HealChannelRequest,
|
||||
response_tx: oneshot::Sender<Result<HealAdmissionResult, String>>,
|
||||
},
|
||||
/// Query heal task status
|
||||
Query { heal_path: String, client_token: String },
|
||||
/// Cancel heal task
|
||||
@@ -339,9 +387,22 @@ pub fn subscribe_heal_responses() -> broadcast::Receiver<HealChannelResponse> {
|
||||
heal_response_sender().subscribe()
|
||||
}
|
||||
|
||||
/// Send heal start request and wait for structured admission feedback.
|
||||
pub async fn send_heal_request_with_admission(request: HealChannelRequest) -> Result<HealAdmissionResult, String> {
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
send_heal_command(HealChannelCommand::Start { request, response_tx }).await?;
|
||||
response_rx
|
||||
.await
|
||||
.map_err(|e| format!("Failed to receive heal admission response: {e}"))?
|
||||
}
|
||||
|
||||
/// Send heal start request
|
||||
pub async fn send_heal_request(request: HealChannelRequest) -> Result<(), String> {
|
||||
send_heal_command(HealChannelCommand::Start(request)).await
|
||||
match send_heal_request_with_admission(request).await? {
|
||||
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => Ok(()),
|
||||
HealAdmissionResult::Full => Err("Heal request queue is full".to_string()),
|
||||
HealAdmissionResult::Dropped(reason) => Err(format!("Heal request dropped: {}", reason.as_str())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Send heal query request
|
||||
@@ -450,7 +511,7 @@ pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str)
|
||||
}
|
||||
|
||||
if let Some(e) = &rule.noncurrent_version_expiration {
|
||||
if let Some(true) = e.noncurrent_days.map(|d| d > 0) {
|
||||
if e.noncurrent_days.is_some() {
|
||||
return true;
|
||||
}
|
||||
if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
|
||||
@@ -542,6 +603,19 @@ pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPri
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn heal_admission_result_labels_are_stable() {
|
||||
assert_eq!(HealAdmissionResult::Accepted.result_label(), "accepted");
|
||||
assert_eq!(HealAdmissionResult::Merged.result_label(), "merged");
|
||||
assert_eq!(HealAdmissionResult::Full.result_label(), "full");
|
||||
assert_eq!(
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull).reason_label(),
|
||||
"queue_full"
|
||||
);
|
||||
assert!(HealAdmissionResult::Merged.is_admitted());
|
||||
assert!(!HealAdmissionResult::Full.is_admitted());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_response_broadcast_reaches_subscriber() {
|
||||
let mut receiver = subscribe_heal_responses();
|
||||
|
||||
@@ -101,11 +101,7 @@ impl InternodeMetrics {
|
||||
pub fn snapshot(&self) -> InternodeMetricsSnapshot {
|
||||
let dial_samples_total = self.dial_samples_total.load(Ordering::Relaxed);
|
||||
let dial_total_time_nanos = self.dial_total_time_nanos.load(Ordering::Relaxed);
|
||||
let dial_avg_time_nanos = if dial_samples_total == 0 {
|
||||
0
|
||||
} else {
|
||||
dial_total_time_nanos / dial_samples_total
|
||||
};
|
||||
let dial_avg_time_nanos = dial_total_time_nanos.checked_div(dial_samples_total).unwrap_or(0);
|
||||
|
||||
InternodeMetricsSnapshot {
|
||||
sent_bytes_total: self.sent_bytes_total.load(Ordering::Relaxed),
|
||||
|
||||
@@ -685,7 +685,7 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn,
|
||||
};
|
||||
|
||||
let done_fn: CloseDiskFn = {
|
||||
let disk = disk_name.clone();
|
||||
let disk = disk_name;
|
||||
Arc::new(move || {
|
||||
let disk = disk.clone();
|
||||
Box::pin(async move {
|
||||
|
||||
@@ -26,7 +26,7 @@ thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
tokio = { workspace = true, features = ["test-util","macros","rt-multi-thread"] }
|
||||
|
||||
[features]
|
||||
default = ["timeout", "lock", "deadlock", "backpressure", "scheduler"]
|
||||
|
||||
@@ -53,8 +53,24 @@ Current guidance:
|
||||
|
||||
## Scanner environment aliases
|
||||
|
||||
- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
|
||||
- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`)
|
||||
- `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
|
||||
- `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
|
||||
- `RUSTFS_SCANNER_IDLE_MODE` (canonical)
|
||||
- `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` (canonical)
|
||||
|
||||
## Drive timeout environment variables
|
||||
|
||||
- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
|
||||
- `RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS`
|
||||
- `RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS`
|
||||
- `RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS`
|
||||
- `RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS`
|
||||
|
||||
Legacy compatibility fallback:
|
||||
- `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION`
|
||||
This legacy variable is treated as a deprecated fallback for the operation-specific drive timeout variables above when a canonical variable is unset.
|
||||
|
||||
## 📄 License
|
||||
|
||||
|
||||
@@ -17,9 +17,13 @@
|
||||
//! webhook and MQTT audit-related settings.
|
||||
|
||||
mod mqtt;
|
||||
mod nats;
|
||||
mod pulsar;
|
||||
mod webhook;
|
||||
|
||||
pub use mqtt::*;
|
||||
pub use nats::*;
|
||||
pub use pulsar::*;
|
||||
pub use webhook::*;
|
||||
|
||||
use crate::DEFAULT_DELIMITER;
|
||||
@@ -30,7 +34,14 @@ pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DE
|
||||
|
||||
pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
|
||||
pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
|
||||
pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats";
|
||||
pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
|
||||
|
||||
pub const AUDIT_STORE_EXTENSION: &str = ".audit";
|
||||
#[allow(dead_code)]
|
||||
pub const AUDIT_SUB_SYSTEMS: &[&str] = &[AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS];
|
||||
pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
|
||||
AUDIT_MQTT_SUB_SYS,
|
||||
AUDIT_NATS_SUB_SYS,
|
||||
AUDIT_PULSAR_SUB_SYS,
|
||||
AUDIT_WEBHOOK_SUB_SYS,
|
||||
];
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub const ENV_AUDIT_NATS_ENABLE: &str = "RUSTFS_AUDIT_NATS_ENABLE";
|
||||
pub const ENV_AUDIT_NATS_ADDRESS: &str = "RUSTFS_AUDIT_NATS_ADDRESS";
|
||||
pub const ENV_AUDIT_NATS_SUBJECT: &str = "RUSTFS_AUDIT_NATS_SUBJECT";
|
||||
pub const ENV_AUDIT_NATS_USERNAME: &str = "RUSTFS_AUDIT_NATS_USERNAME";
|
||||
pub const ENV_AUDIT_NATS_PASSWORD: &str = "RUSTFS_AUDIT_NATS_PASSWORD";
|
||||
pub const ENV_AUDIT_NATS_TOKEN: &str = "RUSTFS_AUDIT_NATS_TOKEN";
|
||||
pub const ENV_AUDIT_NATS_CREDENTIALS_FILE: &str = "RUSTFS_AUDIT_NATS_CREDENTIALS_FILE";
|
||||
pub const ENV_AUDIT_NATS_TLS_CA: &str = "RUSTFS_AUDIT_NATS_TLS_CA";
|
||||
pub const ENV_AUDIT_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_CERT";
|
||||
pub const ENV_AUDIT_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_KEY";
|
||||
pub const ENV_AUDIT_NATS_TLS_REQUIRED: &str = "RUSTFS_AUDIT_NATS_TLS_REQUIRED";
|
||||
pub const ENV_AUDIT_NATS_QUEUE_DIR: &str = "RUSTFS_AUDIT_NATS_QUEUE_DIR";
|
||||
pub const ENV_AUDIT_NATS_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_NATS_QUEUE_LIMIT";
|
||||
|
||||
pub const ENV_AUDIT_NATS_KEYS: &[&str; 13] = &[
|
||||
ENV_AUDIT_NATS_ENABLE,
|
||||
ENV_AUDIT_NATS_ADDRESS,
|
||||
ENV_AUDIT_NATS_SUBJECT,
|
||||
ENV_AUDIT_NATS_USERNAME,
|
||||
ENV_AUDIT_NATS_PASSWORD,
|
||||
ENV_AUDIT_NATS_TOKEN,
|
||||
ENV_AUDIT_NATS_CREDENTIALS_FILE,
|
||||
ENV_AUDIT_NATS_TLS_CA,
|
||||
ENV_AUDIT_NATS_TLS_CLIENT_CERT,
|
||||
ENV_AUDIT_NATS_TLS_CLIENT_KEY,
|
||||
ENV_AUDIT_NATS_TLS_REQUIRED,
|
||||
ENV_AUDIT_NATS_QUEUE_DIR,
|
||||
ENV_AUDIT_NATS_QUEUE_LIMIT,
|
||||
];
|
||||
|
||||
pub const AUDIT_NATS_KEYS: &[&str] = &[
|
||||
crate::ENABLE_KEY,
|
||||
crate::NATS_ADDRESS,
|
||||
crate::NATS_SUBJECT,
|
||||
crate::NATS_USERNAME,
|
||||
crate::NATS_PASSWORD,
|
||||
crate::NATS_TOKEN,
|
||||
crate::NATS_CREDENTIALS_FILE,
|
||||
crate::NATS_TLS_CA,
|
||||
crate::NATS_TLS_CLIENT_CERT,
|
||||
crate::NATS_TLS_CLIENT_KEY,
|
||||
crate::NATS_TLS_REQUIRED,
|
||||
crate::NATS_QUEUE_DIR,
|
||||
crate::NATS_QUEUE_LIMIT,
|
||||
crate::COMMENT_KEY,
|
||||
];
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub const ENV_AUDIT_PULSAR_ENABLE: &str = "RUSTFS_AUDIT_PULSAR_ENABLE";
|
||||
pub const ENV_AUDIT_PULSAR_BROKER: &str = "RUSTFS_AUDIT_PULSAR_BROKER";
|
||||
pub const ENV_AUDIT_PULSAR_TOPIC: &str = "RUSTFS_AUDIT_PULSAR_TOPIC";
|
||||
pub const ENV_AUDIT_PULSAR_AUTH_TOKEN: &str = "RUSTFS_AUDIT_PULSAR_AUTH_TOKEN";
|
||||
pub const ENV_AUDIT_PULSAR_USERNAME: &str = "RUSTFS_AUDIT_PULSAR_USERNAME";
|
||||
pub const ENV_AUDIT_PULSAR_PASSWORD: &str = "RUSTFS_AUDIT_PULSAR_PASSWORD";
|
||||
pub const ENV_AUDIT_PULSAR_TLS_CA: &str = "RUSTFS_AUDIT_PULSAR_TLS_CA";
|
||||
pub const ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_AUDIT_PULSAR_TLS_ALLOW_INSECURE";
|
||||
pub const ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION";
|
||||
pub const ENV_AUDIT_PULSAR_QUEUE_DIR: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_DIR";
|
||||
pub const ENV_AUDIT_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_LIMIT";
|
||||
|
||||
pub const ENV_AUDIT_PULSAR_KEYS: &[&str; 11] = &[
|
||||
ENV_AUDIT_PULSAR_ENABLE,
|
||||
ENV_AUDIT_PULSAR_BROKER,
|
||||
ENV_AUDIT_PULSAR_TOPIC,
|
||||
ENV_AUDIT_PULSAR_AUTH_TOKEN,
|
||||
ENV_AUDIT_PULSAR_USERNAME,
|
||||
ENV_AUDIT_PULSAR_PASSWORD,
|
||||
ENV_AUDIT_PULSAR_TLS_CA,
|
||||
ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE,
|
||||
ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION,
|
||||
ENV_AUDIT_PULSAR_QUEUE_DIR,
|
||||
ENV_AUDIT_PULSAR_QUEUE_LIMIT,
|
||||
];
|
||||
|
||||
pub const AUDIT_PULSAR_KEYS: &[&str] = &[
|
||||
crate::ENABLE_KEY,
|
||||
crate::PULSAR_BROKER,
|
||||
crate::PULSAR_TOPIC,
|
||||
crate::PULSAR_AUTH_TOKEN,
|
||||
crate::PULSAR_USERNAME,
|
||||
crate::PULSAR_PASSWORD,
|
||||
crate::PULSAR_TLS_CA,
|
||||
crate::PULSAR_TLS_ALLOW_INSECURE,
|
||||
crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
|
||||
crate::PULSAR_QUEUE_DIR,
|
||||
crate::PULSAR_QUEUE_LIMIT,
|
||||
crate::COMMENT_KEY,
|
||||
];
|
||||
@@ -39,6 +39,9 @@ pub const ENV_CAPACITY_STAT_TIMEOUT: &str = "RUSTFS_CAPACITY_STAT_TIMEOUT";
|
||||
/// Environment variable for sample rate
|
||||
pub const ENV_CAPACITY_SAMPLE_RATE: &str = "RUSTFS_CAPACITY_SAMPLE_RATE";
|
||||
|
||||
/// Environment variable for metrics logging interval
|
||||
pub const ENV_CAPACITY_METRICS_INTERVAL: &str = "RUSTFS_CAPACITY_METRICS_INTERVAL";
|
||||
|
||||
/// Environment variable for following symbolic links during capacity calculation
|
||||
pub const ENV_CAPACITY_FOLLOW_SYMLINKS: &str = "RUSTFS_CAPACITY_FOLLOW_SYMLINKS";
|
||||
|
||||
@@ -89,6 +92,10 @@ pub const DEFAULT_STAT_TIMEOUT_SECS: u64 = 3;
|
||||
/// Default: 200
|
||||
pub const DEFAULT_SAMPLE_RATE: usize = 200;
|
||||
|
||||
/// Capacity metrics logging interval in seconds
|
||||
/// Default: 600 seconds (10 minutes)
|
||||
pub const DEFAULT_CAPACITY_METRICS_INTERVAL_SECS: u64 = 600;
|
||||
|
||||
/// Follow symbolic links during capacity calculation
|
||||
/// Default: false (disabled for safety)
|
||||
pub const DEFAULT_CAPACITY_FOLLOW_SYMLINKS: bool = false;
|
||||
@@ -130,6 +137,7 @@ mod tests {
|
||||
assert_eq!(ENV_CAPACITY_MAX_FILES_THRESHOLD, "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD");
|
||||
assert_eq!(ENV_CAPACITY_STAT_TIMEOUT, "RUSTFS_CAPACITY_STAT_TIMEOUT");
|
||||
assert_eq!(ENV_CAPACITY_SAMPLE_RATE, "RUSTFS_CAPACITY_SAMPLE_RATE");
|
||||
assert_eq!(ENV_CAPACITY_METRICS_INTERVAL, "RUSTFS_CAPACITY_METRICS_INTERVAL");
|
||||
assert_eq!(ENV_CAPACITY_FOLLOW_SYMLINKS, "RUSTFS_CAPACITY_FOLLOW_SYMLINKS");
|
||||
assert_eq!(ENV_CAPACITY_MAX_SYMLINK_DEPTH, "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH");
|
||||
assert_eq!(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT");
|
||||
@@ -147,6 +155,7 @@ mod tests {
|
||||
assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
|
||||
assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
|
||||
assert_eq!(DEFAULT_SAMPLE_RATE, 200);
|
||||
assert_eq!(DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, 600);
|
||||
assert_eq!(DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, 3);
|
||||
assert_eq!(DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, 2);
|
||||
assert_eq!(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, 15);
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/// Legacy global drive timeout fallback.
|
||||
/// Deprecated in favor of per-operation drive timeout knobs.
|
||||
pub const ENV_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
|
||||
|
||||
/// Default timeout in seconds for the legacy global drive timeout fallback.
|
||||
pub const DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS: u64 = 30;
|
||||
|
||||
/// Timeout for metadata-oriented drive operations such as `read_metadata`.
|
||||
pub const ENV_DRIVE_METADATA_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_METADATA_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_METADATA_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
/// Timeout for `disk_info()` calls on local and remote drives.
|
||||
pub const ENV_DRIVE_DISK_INFO_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
/// Timeout for `list_dir()` style metadata listing operations.
|
||||
pub const ENV_DRIVE_LIST_DIR_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
/// Total timeout for `walk_dir()` operations.
|
||||
pub const ENV_DRIVE_WALKDIR_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
/// Maximum time without forward progress while consuming a `walk_dir()` stream.
|
||||
pub const ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
/// Number of consecutive failures before a suspect drive is classified as offline.
|
||||
pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
|
||||
pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;
|
||||
|
||||
/// Number of consecutive successful recovery probes before a returning drive is considered online again.
|
||||
pub const ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD: &str = "RUSTFS_DRIVE_RETURNING_SUCCESS_THRESHOLD";
|
||||
pub const DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD: u64 = 3;
|
||||
|
||||
/// Probe interval in seconds while a drive is in the recovery path.
|
||||
pub const ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS: &str = "RUSTFS_DRIVE_RETURNING_PROBE_INTERVAL_SECS";
|
||||
pub const DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS: u64 = 2;
|
||||
|
||||
/// Duration in seconds for classifying a recovered drive as a short offline event.
|
||||
pub const ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS: &str = "RUSTFS_DRIVE_OFFLINE_GRACE_PERIOD_SECS";
|
||||
pub const DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS: u64 = 30;
|
||||
|
||||
/// Duration in seconds after which a recovered drive is classified as long offline.
|
||||
pub const ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: &str = "RUSTFS_DRIVE_LONG_OFFLINE_THRESHOLD_SECS";
|
||||
pub const DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: u64 = 172_800;
|
||||
@@ -56,6 +56,15 @@ pub const ENV_HEAL_TASK_TIMEOUT_SECS: &str = "RUSTFS_HEAL_TASK_TIMEOUT_SECS";
|
||||
/// - Note: A higher concurrency limit can speed up healing but may lead to resource contention.
|
||||
pub const ENV_HEAL_MAX_CONCURRENT_HEALS: &str = "RUSTFS_HEAL_MAX_CONCURRENT_HEALS";
|
||||
|
||||
/// Environment variable name that specifies the maximum number of concurrent heal operations
|
||||
/// allowed for a single erasure set.
|
||||
///
|
||||
/// - Purpose: Prevent one degraded set from consuming all global heal slots.
|
||||
/// - Unit: number of operations (usize).
|
||||
/// - Valid values: any positive integer.
|
||||
/// - Example: `export RUSTFS_HEAL_MAX_CONCURRENT_PER_SET=1`
|
||||
pub const ENV_HEAL_MAX_CONCURRENT_PER_SET: &str = "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET";
|
||||
|
||||
/// Default value for enabling authentication for heal operations if not specified in the environment variable.
|
||||
/// - Value: true (authentication enabled).
|
||||
/// - Rationale: Enabling authentication by default enhances security for heal operations.
|
||||
@@ -86,3 +95,46 @@ pub const DEFAULT_HEAL_TASK_TIMEOUT_SECS: u64 = 300; // 5 minutes
|
||||
/// - Rationale: This default concurrency limit helps balance healing speed with resource usage, preventing system overload.
|
||||
/// - Adjustments: Users may modify this value via the `RUSTFS_HEAL_MAX_CONCURRENT_HEALS` environment variable based on their system capacity and expected heal workload.
|
||||
pub const DEFAULT_HEAL_MAX_CONCURRENT_HEALS: usize = 4;
|
||||
|
||||
/// Default maximum number of concurrent heal operations per erasure set.
|
||||
///
|
||||
/// - Value: 1 concurrent heal operation per set.
|
||||
/// - Rationale: Keeps a degraded set from monopolizing the global heal scheduler.
|
||||
pub const DEFAULT_HEAL_MAX_CONCURRENT_PER_SET: usize = 1;
|
||||
|
||||
/// Environment variable that controls whether low-priority heal requests should merge into
|
||||
/// an existing queued request with the same deduplication key.
|
||||
pub const ENV_HEAL_LOW_PRIORITY_MERGE_ENABLE: &str = "RUSTFS_HEAL_LOW_PRIORITY_MERGE_ENABLE";
|
||||
|
||||
/// Environment variable that allows low-priority heal requests to be dropped when the queue is full.
|
||||
pub const ENV_HEAL_LOW_PRIORITY_DROP_WHEN_FULL: &str = "RUSTFS_HEAL_LOW_PRIORITY_DROP_WHEN_FULL";
|
||||
|
||||
/// Environment variable that controls concurrent object heals within a single erasure-set page.
|
||||
pub const ENV_HEAL_PAGE_OBJECT_CONCURRENCY: &str = "RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY";
|
||||
|
||||
/// Environment variable that toggles notify-driven scheduler wakeups.
|
||||
pub const ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE: &str = "RUSTFS_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE";
|
||||
|
||||
/// Environment variable that toggles per-set bulkhead scheduling.
|
||||
pub const ENV_HEAL_SET_BULKHEAD_ENABLE: &str = "RUSTFS_HEAL_SET_BULKHEAD_ENABLE";
|
||||
|
||||
/// Environment variable that toggles page-level parallel object healing for erasure-set repair.
|
||||
pub const ENV_HEAL_PAGE_PARALLEL_ENABLE: &str = "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE";
|
||||
|
||||
/// Default behavior is to merge duplicate low-priority requests.
|
||||
pub const DEFAULT_HEAL_LOW_PRIORITY_MERGE_ENABLE: bool = true;
|
||||
|
||||
/// Default behavior is to drop low-priority requests instead of blocking when the queue is full.
|
||||
pub const DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL: bool = true;
|
||||
|
||||
/// Default per-page object heal concurrency for erasure-set healing.
|
||||
pub const DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY: usize = 8;
|
||||
|
||||
/// Default behavior is to keep notify-driven scheduler wakeups enabled.
|
||||
pub const DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE: bool = true;
|
||||
|
||||
/// Default behavior is to keep per-set bulkhead scheduling enabled.
|
||||
pub const DEFAULT_HEAL_SET_BULKHEAD_ENABLE: bool = true;
|
||||
|
||||
/// Default behavior is to keep erasure-set page parallelism enabled.
|
||||
pub const DEFAULT_HEAL_PAGE_PARALLEL_ENABLE: bool = true;
|
||||
|
||||
@@ -17,6 +17,7 @@ pub(crate) mod body_limits;
|
||||
pub(crate) mod capacity;
|
||||
pub(crate) mod compress;
|
||||
pub(crate) mod console;
|
||||
pub(crate) mod drive;
|
||||
pub(crate) mod env;
|
||||
pub(crate) mod heal;
|
||||
pub(crate) mod object;
|
||||
|
||||
@@ -17,6 +17,7 @@ pub const OIDC_CONFIG_URL: &str = "config_url";
|
||||
pub const OIDC_CLIENT_ID: &str = "client_id";
|
||||
pub const OIDC_CLIENT_SECRET: &str = "client_secret";
|
||||
pub const OIDC_SCOPES: &str = "scopes";
|
||||
pub const OIDC_OTHER_AUDIENCES: &str = "other_audiences";
|
||||
pub const OIDC_REDIRECT_URI: &str = "redirect_uri";
|
||||
pub const OIDC_REDIRECT_URI_DYNAMIC: &str = "redirect_uri_dynamic";
|
||||
pub const OIDC_CLAIM_NAME: &str = "claim_name";
|
||||
@@ -24,6 +25,7 @@ pub const OIDC_CLAIM_PREFIX: &str = "claim_prefix";
|
||||
pub const OIDC_ROLE_POLICY: &str = "role_policy";
|
||||
pub const OIDC_DISPLAY_NAME: &str = "display_name";
|
||||
pub const OIDC_GROUPS_CLAIM: &str = "groups_claim";
|
||||
pub const OIDC_ROLES_CLAIM: &str = "roles_claim";
|
||||
pub const OIDC_EMAIL_CLAIM: &str = "email_claim";
|
||||
pub const OIDC_USERNAME_CLAIM: &str = "username_claim";
|
||||
|
||||
@@ -33,6 +35,7 @@ pub const ENV_IDENTITY_OPENID_CONFIG_URL: &str = "RUSTFS_IDENTITY_OPENID_CONFIG_
|
||||
pub const ENV_IDENTITY_OPENID_CLIENT_ID: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_ID";
|
||||
pub const ENV_IDENTITY_OPENID_CLIENT_SECRET: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_SECRET";
|
||||
pub const ENV_IDENTITY_OPENID_SCOPES: &str = "RUSTFS_IDENTITY_OPENID_SCOPES";
|
||||
pub const ENV_IDENTITY_OPENID_OTHER_AUDIENCES: &str = "RUSTFS_IDENTITY_OPENID_OTHER_AUDIENCES";
|
||||
pub const ENV_IDENTITY_OPENID_REDIRECT_URI: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI";
|
||||
pub const ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC";
|
||||
pub const ENV_IDENTITY_OPENID_CLAIM_NAME: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_NAME";
|
||||
@@ -40,16 +43,18 @@ pub const ENV_IDENTITY_OPENID_CLAIM_PREFIX: &str = "RUSTFS_IDENTITY_OPENID_CLAIM
|
||||
pub const ENV_IDENTITY_OPENID_ROLE_POLICY: &str = "RUSTFS_IDENTITY_OPENID_ROLE_POLICY";
|
||||
pub const ENV_IDENTITY_OPENID_DISPLAY_NAME: &str = "RUSTFS_IDENTITY_OPENID_DISPLAY_NAME";
|
||||
pub const ENV_IDENTITY_OPENID_GROUPS_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM";
|
||||
pub const ENV_IDENTITY_OPENID_ROLES_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_ROLES_CLAIM";
|
||||
pub const ENV_IDENTITY_OPENID_EMAIL_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM";
|
||||
pub const ENV_IDENTITY_OPENID_USERNAME_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM";
|
||||
|
||||
/// List of all environment variable keys for an OIDC provider.
|
||||
pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
|
||||
pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 16] = &[
|
||||
ENV_IDENTITY_OPENID_ENABLE,
|
||||
ENV_IDENTITY_OPENID_CONFIG_URL,
|
||||
ENV_IDENTITY_OPENID_CLIENT_ID,
|
||||
ENV_IDENTITY_OPENID_CLIENT_SECRET,
|
||||
ENV_IDENTITY_OPENID_SCOPES,
|
||||
ENV_IDENTITY_OPENID_OTHER_AUDIENCES,
|
||||
ENV_IDENTITY_OPENID_REDIRECT_URI,
|
||||
ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC,
|
||||
ENV_IDENTITY_OPENID_CLAIM_NAME,
|
||||
@@ -57,6 +62,7 @@ pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
|
||||
ENV_IDENTITY_OPENID_ROLE_POLICY,
|
||||
ENV_IDENTITY_OPENID_DISPLAY_NAME,
|
||||
ENV_IDENTITY_OPENID_GROUPS_CLAIM,
|
||||
ENV_IDENTITY_OPENID_ROLES_CLAIM,
|
||||
ENV_IDENTITY_OPENID_EMAIL_CLAIM,
|
||||
ENV_IDENTITY_OPENID_USERNAME_CLAIM,
|
||||
];
|
||||
@@ -68,6 +74,7 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
|
||||
OIDC_CLIENT_ID,
|
||||
OIDC_CLIENT_SECRET,
|
||||
OIDC_SCOPES,
|
||||
OIDC_OTHER_AUDIENCES,
|
||||
OIDC_REDIRECT_URI,
|
||||
OIDC_REDIRECT_URI_DYNAMIC,
|
||||
OIDC_CLAIM_NAME,
|
||||
@@ -75,6 +82,7 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
|
||||
OIDC_ROLE_POLICY,
|
||||
OIDC_DISPLAY_NAME,
|
||||
OIDC_GROUPS_CLAIM,
|
||||
OIDC_ROLES_CLAIM,
|
||||
OIDC_EMAIL_CLAIM,
|
||||
OIDC_USERNAME_CLAIM,
|
||||
crate::COMMENT_KEY,
|
||||
@@ -84,6 +92,8 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
|
||||
pub const OIDC_DEFAULT_SCOPES: &str = "openid,profile,email";
|
||||
pub const OIDC_DEFAULT_CLAIM_NAME: &str = "groups";
|
||||
pub const OIDC_DEFAULT_GROUPS_CLAIM: &str = "groups";
|
||||
/// Empty means do not merge a secondary claim into groups (legacy behavior). Set to e.g. `roles` to opt in.
|
||||
pub const OIDC_DEFAULT_ROLES_CLAIM: &str = "";
|
||||
pub const OIDC_DEFAULT_EMAIL_CLAIM: &str = "email";
|
||||
pub const OIDC_DEFAULT_USERNAME_CLAIM: &str = "preferred_username";
|
||||
|
||||
|
||||
@@ -25,6 +25,12 @@ pub const ENV_SCANNER_START_DELAY_SECS: &str = "RUSTFS_SCANNER_START_DELAY_SECS"
|
||||
#[deprecated(note = "Use RUSTFS_SCANNER_START_DELAY_SECS instead")]
|
||||
pub const ENV_DATA_SCANNER_START_DELAY_SECS: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
|
||||
|
||||
/// Environment variable that specifies the scanner cycle interval in seconds.
|
||||
/// If set, this overrides the cycle interval derived from `RUSTFS_SCANNER_SPEED`.
|
||||
/// - Unit: seconds (u64).
|
||||
/// - Example: `export RUSTFS_SCANNER_CYCLE=3600` (1 hour)
|
||||
pub const ENV_SCANNER_CYCLE: &str = "RUSTFS_SCANNER_CYCLE";
|
||||
|
||||
/// Environment variable that selects the scanner speed preset.
|
||||
/// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
|
||||
/// Controls the sleep factor, maximum sleep duration, and cycle interval.
|
||||
@@ -39,9 +45,24 @@ pub const DEFAULT_SCANNER_SPEED: &str = "default";
|
||||
/// - Example: `export RUSTFS_SCANNER_IDLE_MODE=false`
|
||||
pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE";
|
||||
|
||||
/// Environment variable that controls scanner cache save timeout in seconds.
|
||||
/// The scanner enforces a minimum value of `1`.
|
||||
/// - Unit: seconds (u64).
|
||||
/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30`
|
||||
pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS";
|
||||
|
||||
/// Default scanner idle mode.
|
||||
pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
|
||||
|
||||
/// Compatibility flag kept for Patch 3 rollback windows.
|
||||
///
|
||||
/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue.
|
||||
/// When this flag is enabled, RustFS logs a warning and continues to use enqueue-based heal.
|
||||
pub const ENV_SCANNER_INLINE_HEAL_ENABLE: &str = "RUSTFS_SCANNER_INLINE_HEAL_ENABLE";
|
||||
|
||||
/// Default inline scanner heal compatibility mode.
|
||||
pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
|
||||
|
||||
/// Scanner speed preset controlling throttling behavior.
|
||||
///
|
||||
/// Each preset defines three parameters:
|
||||
@@ -56,6 +77,8 @@ pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
|
||||
/// | `default` | 2x | 1 second | 1 minute |
|
||||
/// | `slow` | 10x | 15 seconds| 1 minute |
|
||||
/// | `slowest` | 100x | 15 seconds| 30 minutes |
|
||||
///
|
||||
/// The cycle interval can be overridden by `RUSTFS_SCANNER_CYCLE`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum ScannerSpeed {
|
||||
Fastest,
|
||||
|
||||
@@ -41,6 +41,30 @@ pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
|
||||
pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
|
||||
pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
|
||||
|
||||
pub const NATS_ADDRESS: &str = "address";
|
||||
pub const NATS_SUBJECT: &str = "subject";
|
||||
pub const NATS_USERNAME: &str = "username";
|
||||
pub const NATS_PASSWORD: &str = "password";
|
||||
pub const NATS_TOKEN: &str = "token";
|
||||
pub const NATS_CREDENTIALS_FILE: &str = "credentials_file";
|
||||
pub const NATS_TLS_CA: &str = "tls_ca";
|
||||
pub const NATS_TLS_CLIENT_CERT: &str = "tls_client_cert";
|
||||
pub const NATS_TLS_CLIENT_KEY: &str = "tls_client_key";
|
||||
pub const NATS_TLS_REQUIRED: &str = "tls_required";
|
||||
pub const NATS_QUEUE_DIR: &str = "queue_dir";
|
||||
pub const NATS_QUEUE_LIMIT: &str = "queue_limit";
|
||||
|
||||
pub const PULSAR_BROKER: &str = "broker";
|
||||
pub const PULSAR_TOPIC: &str = "topic";
|
||||
pub const PULSAR_AUTH_TOKEN: &str = "auth_token";
|
||||
pub const PULSAR_USERNAME: &str = "username";
|
||||
pub const PULSAR_PASSWORD: &str = "password";
|
||||
pub const PULSAR_TLS_CA: &str = "tls_ca";
|
||||
pub const PULSAR_TLS_ALLOW_INSECURE: &str = "tls_allow_insecure";
|
||||
pub const PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "tls_hostname_verification";
|
||||
pub const PULSAR_QUEUE_DIR: &str = "queue_dir";
|
||||
pub const PULSAR_QUEUE_LIMIT: &str = "queue_limit";
|
||||
|
||||
/// Environment variable controlling whether target queue files are Snappy-compressed.
|
||||
/// Applies to both notify and audit target queue stores.
|
||||
pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
|
||||
|
||||
@@ -49,34 +49,6 @@ pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE";
|
||||
/// to regular I/O without errors.
|
||||
pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = true;
|
||||
|
||||
/// Environment variable for zero-copy read operating mode.
|
||||
///
|
||||
/// Supported values:
|
||||
/// - `off`: disable mmap-backed chunk fast path and always use the compatibility path
|
||||
/// - `conservative`: allow a single mmap window per request
|
||||
/// - `balanced`: allow multiple mmap windows with the default size guardrails
|
||||
/// - `aggressive`: allow multi-window mmap and relax the small-object cutoff
|
||||
pub const ENV_OBJECT_ZERO_COPY_MODE: &str = "RUSTFS_OBJECT_ZERO_COPY_MODE";
|
||||
|
||||
/// Default zero-copy read mode.
|
||||
pub const DEFAULT_OBJECT_ZERO_COPY_MODE: &str = "balanced";
|
||||
|
||||
/// Environment variable for the maximum mmap window size used by the chunk fast path.
|
||||
///
|
||||
/// This controls the visible bytes per mapped chunk before the implementation emits a new window.
|
||||
pub const ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES";
|
||||
|
||||
/// Default mmap window size for chunk fast path reads: 8 MiB.
|
||||
pub const DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Environment variable for the maximum total active mmap bytes.
|
||||
///
|
||||
/// Requests that would exceed this active window budget fall back to the compatibility path.
|
||||
pub const ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES";
|
||||
|
||||
/// Default maximum active mmap bytes across concurrent local chunk fast-path reads: 256 MiB.
|
||||
pub const DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: usize = 256 * 1024 * 1024;
|
||||
|
||||
// =============================================================================
|
||||
// Direct I/O Configuration
|
||||
// =============================================================================
|
||||
|
||||
@@ -25,6 +25,8 @@ pub use constants::compress::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::console::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::drive::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::env::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::heal::*;
|
||||
|
||||
@@ -14,11 +14,15 @@
|
||||
|
||||
mod arn;
|
||||
mod mqtt;
|
||||
mod nats;
|
||||
mod pulsar;
|
||||
mod store;
|
||||
mod webhook;
|
||||
|
||||
pub use arn::*;
|
||||
pub use mqtt::*;
|
||||
pub use nats::*;
|
||||
pub use pulsar::*;
|
||||
pub use store::*;
|
||||
pub use webhook::*;
|
||||
|
||||
@@ -64,7 +68,12 @@ pub const ENV_NOTIFY_SEND_CONCURRENCY: &str = "RUSTFS_NOTIFY_SEND_CONCURRENCY";
|
||||
pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS];
|
||||
pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
|
||||
NOTIFY_MQTT_SUB_SYS,
|
||||
NOTIFY_NATS_SUB_SYS,
|
||||
NOTIFY_PULSAR_SUB_SYS,
|
||||
NOTIFY_WEBHOOK_SUB_SYS,
|
||||
];
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
|
||||
@@ -83,4 +92,5 @@ pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
|
||||
pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
|
||||
pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
|
||||
pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub const NOTIFY_NATS_KEYS: &[&str] = &[
|
||||
crate::ENABLE_KEY,
|
||||
crate::NATS_ADDRESS,
|
||||
crate::NATS_SUBJECT,
|
||||
crate::NATS_USERNAME,
|
||||
crate::NATS_PASSWORD,
|
||||
crate::NATS_TOKEN,
|
||||
crate::NATS_CREDENTIALS_FILE,
|
||||
crate::NATS_TLS_CA,
|
||||
crate::NATS_TLS_CLIENT_CERT,
|
||||
crate::NATS_TLS_CLIENT_KEY,
|
||||
crate::NATS_TLS_REQUIRED,
|
||||
crate::NATS_QUEUE_DIR,
|
||||
crate::NATS_QUEUE_LIMIT,
|
||||
crate::COMMENT_KEY,
|
||||
];
|
||||
|
||||
pub const ENV_NOTIFY_NATS_ENABLE: &str = "RUSTFS_NOTIFY_NATS_ENABLE";
|
||||
pub const ENV_NOTIFY_NATS_ADDRESS: &str = "RUSTFS_NOTIFY_NATS_ADDRESS";
|
||||
pub const ENV_NOTIFY_NATS_SUBJECT: &str = "RUSTFS_NOTIFY_NATS_SUBJECT";
|
||||
pub const ENV_NOTIFY_NATS_USERNAME: &str = "RUSTFS_NOTIFY_NATS_USERNAME";
|
||||
pub const ENV_NOTIFY_NATS_PASSWORD: &str = "RUSTFS_NOTIFY_NATS_PASSWORD";
|
||||
pub const ENV_NOTIFY_NATS_TOKEN: &str = "RUSTFS_NOTIFY_NATS_TOKEN";
|
||||
pub const ENV_NOTIFY_NATS_CREDENTIALS_FILE: &str = "RUSTFS_NOTIFY_NATS_CREDENTIALS_FILE";
|
||||
pub const ENV_NOTIFY_NATS_TLS_CA: &str = "RUSTFS_NOTIFY_NATS_TLS_CA";
|
||||
pub const ENV_NOTIFY_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_CERT";
|
||||
pub const ENV_NOTIFY_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_KEY";
|
||||
pub const ENV_NOTIFY_NATS_TLS_REQUIRED: &str = "RUSTFS_NOTIFY_NATS_TLS_REQUIRED";
|
||||
pub const ENV_NOTIFY_NATS_QUEUE_DIR: &str = "RUSTFS_NOTIFY_NATS_QUEUE_DIR";
|
||||
pub const ENV_NOTIFY_NATS_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_NATS_QUEUE_LIMIT";
|
||||
|
||||
pub const ENV_NOTIFY_NATS_KEYS: &[&str; 13] = &[
|
||||
ENV_NOTIFY_NATS_ENABLE,
|
||||
ENV_NOTIFY_NATS_ADDRESS,
|
||||
ENV_NOTIFY_NATS_SUBJECT,
|
||||
ENV_NOTIFY_NATS_USERNAME,
|
||||
ENV_NOTIFY_NATS_PASSWORD,
|
||||
ENV_NOTIFY_NATS_TOKEN,
|
||||
ENV_NOTIFY_NATS_CREDENTIALS_FILE,
|
||||
ENV_NOTIFY_NATS_TLS_CA,
|
||||
ENV_NOTIFY_NATS_TLS_CLIENT_CERT,
|
||||
ENV_NOTIFY_NATS_TLS_CLIENT_KEY,
|
||||
ENV_NOTIFY_NATS_TLS_REQUIRED,
|
||||
ENV_NOTIFY_NATS_QUEUE_DIR,
|
||||
ENV_NOTIFY_NATS_QUEUE_LIMIT,
|
||||
];
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub const NOTIFY_PULSAR_KEYS: &[&str] = &[
|
||||
crate::ENABLE_KEY,
|
||||
crate::PULSAR_BROKER,
|
||||
crate::PULSAR_TOPIC,
|
||||
crate::PULSAR_AUTH_TOKEN,
|
||||
crate::PULSAR_USERNAME,
|
||||
crate::PULSAR_PASSWORD,
|
||||
crate::PULSAR_TLS_CA,
|
||||
crate::PULSAR_TLS_ALLOW_INSECURE,
|
||||
crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
|
||||
crate::PULSAR_QUEUE_DIR,
|
||||
crate::PULSAR_QUEUE_LIMIT,
|
||||
crate::COMMENT_KEY,
|
||||
];
|
||||
|
||||
pub const ENV_NOTIFY_PULSAR_ENABLE: &str = "RUSTFS_NOTIFY_PULSAR_ENABLE";
|
||||
pub const ENV_NOTIFY_PULSAR_BROKER: &str = "RUSTFS_NOTIFY_PULSAR_BROKER";
|
||||
pub const ENV_NOTIFY_PULSAR_TOPIC: &str = "RUSTFS_NOTIFY_PULSAR_TOPIC";
|
||||
pub const ENV_NOTIFY_PULSAR_AUTH_TOKEN: &str = "RUSTFS_NOTIFY_PULSAR_AUTH_TOKEN";
|
||||
pub const ENV_NOTIFY_PULSAR_USERNAME: &str = "RUSTFS_NOTIFY_PULSAR_USERNAME";
|
||||
pub const ENV_NOTIFY_PULSAR_PASSWORD: &str = "RUSTFS_NOTIFY_PULSAR_PASSWORD";
|
||||
pub const ENV_NOTIFY_PULSAR_TLS_CA: &str = "RUSTFS_NOTIFY_PULSAR_TLS_CA";
|
||||
pub const ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_NOTIFY_PULSAR_TLS_ALLOW_INSECURE";
|
||||
pub const ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION";
|
||||
pub const ENV_NOTIFY_PULSAR_QUEUE_DIR: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_DIR";
|
||||
pub const ENV_NOTIFY_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_LIMIT";
|
||||
|
||||
pub const ENV_NOTIFY_PULSAR_KEYS: &[&str; 11] = &[
|
||||
ENV_NOTIFY_PULSAR_ENABLE,
|
||||
ENV_NOTIFY_PULSAR_BROKER,
|
||||
ENV_NOTIFY_PULSAR_TOPIC,
|
||||
ENV_NOTIFY_PULSAR_AUTH_TOKEN,
|
||||
ENV_NOTIFY_PULSAR_USERNAME,
|
||||
ENV_NOTIFY_PULSAR_PASSWORD,
|
||||
ENV_NOTIFY_PULSAR_TLS_CA,
|
||||
ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE,
|
||||
ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION,
|
||||
ENV_NOTIFY_PULSAR_QUEUE_DIR,
|
||||
ENV_NOTIFY_PULSAR_QUEUE_LIMIT,
|
||||
];
|
||||
@@ -20,7 +20,7 @@ use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::fmt;
|
||||
use std::io::Error;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::{LazyLock, OnceLock};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Global active credentials
|
||||
@@ -204,10 +204,11 @@ pub fn gen_secret_key(length: usize) -> std::io::Result<String> {
|
||||
let mut key = vec![0u8; URL_SAFE_NO_PAD.estimated_decoded_length(length)];
|
||||
rng.fill_bytes(&mut key);
|
||||
|
||||
// URL_SAFE_NO_PAD uses "-" and "_" instead of "+" and "/", so "/" never
|
||||
// appears in the output. The .replace("/", "+") was a dead no-op.
|
||||
let encoded = URL_SAFE_NO_PAD.encode_to_string(&key);
|
||||
let key_str = encoded.replace("/", "+");
|
||||
|
||||
Ok(key_str)
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
/// Get the RPC authentication token from environment variable
|
||||
@@ -313,6 +314,17 @@ impl fmt::Debug for Credentials {
|
||||
}
|
||||
|
||||
impl Credentials {
|
||||
/// Returns a reference to this credential's claims, or a shared empty map
|
||||
/// when the credential has no claims attached. Avoids per-call allocation
|
||||
/// at call sites that need an `&HashMap<String, Value>`.
|
||||
pub fn claims_or_empty(&self) -> &HashMap<String, Value> {
|
||||
static EMPTY: LazyLock<HashMap<String, Value>> = LazyLock::new(HashMap::new);
|
||||
match &self.claims {
|
||||
Some(c) => c,
|
||||
None => &EMPTY,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_expired(&self) -> bool {
|
||||
if self.expiration.is_none() {
|
||||
return false;
|
||||
@@ -447,7 +459,7 @@ mod tests {
|
||||
// Initialize
|
||||
let test_ak = "test_access_key".to_string();
|
||||
let test_sk = "test_secret_key_123456".to_string();
|
||||
init_global_action_credentials(Some(test_ak.clone()), Some(test_sk.clone())).ok();
|
||||
init_global_action_credentials(Some(test_ak), Some(test_sk)).ok();
|
||||
}
|
||||
|
||||
// Verify the state after initialization
|
||||
@@ -473,6 +485,16 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gen_secret_key_uses_url_safe_base64_without_padding() {
|
||||
let key = gen_secret_key(32).expect("secret key should generate");
|
||||
|
||||
assert_eq!(key.len(), 32);
|
||||
assert!(!key.contains('/'));
|
||||
assert!(!key.contains('+'));
|
||||
assert!(!key.contains('='));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_masked_debug() {
|
||||
// Test None
|
||||
|
||||
@@ -20,11 +20,6 @@ license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "small_put_bench"
|
||||
path = "src/bin/small_put_bench.rs"
|
||||
test = false
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -46,7 +41,6 @@ serde_json.workspace = true
|
||||
tonic = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tokio-stream = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes.workspace = true
|
||||
@@ -58,8 +52,6 @@ async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
|
||||
async-trait = { workspace = true }
|
||||
flate2.workspace = true
|
||||
http.workspace = true
|
||||
http-body.workspace = true
|
||||
http-body-util.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
rustfs-signer.workspace = true
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -14,7 +14,9 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path};
|
||||
use aws_sdk_s3::Client as S3Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
@@ -28,6 +30,8 @@ mod tests {
|
||||
use std::io::{Cursor, Write};
|
||||
use std::process::Command;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions};
|
||||
|
||||
const ARCHIVE_TEST_BUCKET: &str = "archive-download-integrity";
|
||||
@@ -116,6 +120,75 @@ mod tests {
|
||||
.await?)
|
||||
}
|
||||
|
||||
fn find_header_terminator(buf: &[u8]) -> Option<usize> {
|
||||
buf.windows(4).position(|window| window == b"\r\n\r\n")
|
||||
}
|
||||
|
||||
async fn read_proxy_request(stream: &mut tokio::net::TcpStream) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let mut buffer = Vec::new();
|
||||
let mut chunk = [0_u8; 4096];
|
||||
|
||||
loop {
|
||||
let read = stream.read(&mut chunk).await?;
|
||||
if read == 0 {
|
||||
return Err("proxy request ended before headers were fully received".into());
|
||||
}
|
||||
buffer.extend_from_slice(&chunk[..read]);
|
||||
if find_header_terminator(&buffer).is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn_reverse_proxy_to_presigned_url(
|
||||
target_url: String,
|
||||
) -> Result<(String, tokio::task::JoinHandle<Result<(), Box<dyn Error + Send + Sync>>>), Box<dyn Error + Send + Sync>> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let address = listener.local_addr()?;
|
||||
let proxy_url = format!("http://{address}/");
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let (mut downstream, _) = listener.accept().await?;
|
||||
read_proxy_request(&mut downstream).await?;
|
||||
|
||||
let upstream_response: Result<reqwest::Response, reqwest::Error> = local_http_client().get(&target_url).send().await;
|
||||
let (status, body, content_type) = match upstream_response {
|
||||
Ok(response) => {
|
||||
let status = response.status();
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get("content-type")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_string);
|
||||
match response.bytes().await {
|
||||
Ok(body) => (status, body.to_vec(), content_type),
|
||||
Err(err) => {
|
||||
let body = format!("upstream body read failed: {err}").into_bytes();
|
||||
(StatusCode::BAD_GATEWAY, body, Some("text/plain".to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let body = format!("upstream request failed: {err}").into_bytes();
|
||||
(StatusCode::BAD_GATEWAY, body, Some("text/plain".to_string()))
|
||||
}
|
||||
};
|
||||
|
||||
let mut response_head = format!("HTTP/1.1 {}\r\ncontent-length: {}\r\nconnection: close\r\n", status, body.len());
|
||||
if let Some(content_type) = content_type {
|
||||
response_head.push_str(&format!("content-type: {content_type}\r\n"));
|
||||
}
|
||||
response_head.push_str("\r\n");
|
||||
|
||||
downstream.write_all(response_head.as_bytes()).await?;
|
||||
downstream.write_all(&body).await?;
|
||||
downstream.shutdown().await?;
|
||||
Ok(())
|
||||
});
|
||||
|
||||
Ok((proxy_url, handle))
|
||||
}
|
||||
|
||||
async fn signed_put_request_with_headers(
|
||||
url: &str,
|
||||
access_key: &str,
|
||||
@@ -145,12 +218,157 @@ mod tests {
|
||||
Ok(builder.send().await?)
|
||||
}
|
||||
|
||||
async fn signed_get_request_with_headers(
|
||||
url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
extra_headers: &[(&str, &str)],
|
||||
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
|
||||
let mut request = http::Request::builder()
|
||||
.method(http::Method::GET)
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
for (name, value) in extra_headers {
|
||||
request = request.header(*name, *value);
|
||||
}
|
||||
|
||||
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let client = local_http_client();
|
||||
let mut builder = client.get(url);
|
||||
for (name, value) in signed.headers() {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
|
||||
Ok(builder.send().await?)
|
||||
}
|
||||
|
||||
async fn assert_archive_object_content_encoding(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
expected_content_encoding: Option<&str>,
|
||||
expected_body: &[u8],
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let head_resp = client.head_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(head_resp.content_encoding(), expected_content_encoding);
|
||||
|
||||
let get_resp = client.get_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(get_resp.content_encoding(), expected_content_encoding);
|
||||
let body = get_resp.body.collect().await?.into_bytes();
|
||||
assert_eq!(body.as_ref(), expected_body);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn complete_archive_multipart_upload_with_content_encoding(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
content_encoding: Option<&str>,
|
||||
) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let payload = random_bytes(MULTIPART_PART_SIZE + 256 * 1024);
|
||||
let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?;
|
||||
assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold");
|
||||
|
||||
let mut create_builder = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.content_type("application/zip");
|
||||
if let Some(content_encoding) = content_encoding {
|
||||
create_builder = create_builder.content_encoding(content_encoding);
|
||||
}
|
||||
let create_output = create_builder.send().await?;
|
||||
let upload_id = create_output.upload_id().expect("multipart upload id");
|
||||
|
||||
let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec();
|
||||
let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec();
|
||||
|
||||
let upload_part_1 = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(first_part))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let upload_part_2 = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(second_part))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let completed_upload = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(upload_part_1.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(upload_part_2.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed_upload)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
Ok(zip_bytes)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_archive_put_rejects_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
async fn test_archive_put_allows_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
start_rustfs_server_with_env(&mut env, &[]).await?;
|
||||
env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
|
||||
let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
|
||||
let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
|
||||
let response =
|
||||
signed_put_request_with_headers(&object_url, &env.access_key, &env.secret_key, zip_bytes, "application/zip", "gzip")
|
||||
.await?;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let head_resp = client
|
||||
.head_object()
|
||||
.bucket(ARCHIVE_TEST_BUCKET)
|
||||
.key("bundle.zip")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(head_resp.content_encoding(), Some("gzip"));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_archive_put_rejects_content_encoding_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
|
||||
env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
|
||||
let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
|
||||
let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
|
||||
@@ -161,7 +379,145 @@ mod tests {
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = response.text().await?;
|
||||
assert!(
|
||||
body.contains("InvalidArgument") || body.contains("Content-Encoding"),
|
||||
body.contains("InvalidArgument") || body.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING"),
|
||||
"unexpected error body: {body}"
|
||||
);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_archive_put_with_aws_chunked_does_not_persist_content_encoding_by_default()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_server_with_env(&mut env, &[]).await?;
|
||||
env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
|
||||
|
||||
let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
|
||||
let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-aws-chunked.zip");
|
||||
let response = signed_put_request_with_headers(
|
||||
&object_url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
zip_bytes.clone(),
|
||||
"application/zip",
|
||||
"aws-chunked",
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let client = env.create_s3_client();
|
||||
assert_archive_object_content_encoding(
|
||||
&client,
|
||||
ARCHIVE_TEST_BUCKET,
|
||||
"bundle-aws-chunked.zip",
|
||||
None,
|
||||
zip_bytes.as_slice(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_archive_put_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_server_with_env(&mut env, &[]).await?;
|
||||
env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
|
||||
|
||||
let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
|
||||
let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-aws-chunked-gzip.zip");
|
||||
let response = signed_put_request_with_headers(
|
||||
&object_url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
zip_bytes.clone(),
|
||||
"application/zip",
|
||||
"aws-chunked,gzip",
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let client = env.create_s3_client();
|
||||
assert_archive_object_content_encoding(
|
||||
&client,
|
||||
ARCHIVE_TEST_BUCKET,
|
||||
"bundle-aws-chunked-gzip.zip",
|
||||
Some("gzip"),
|
||||
zip_bytes.as_slice(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_archive_put_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
|
||||
env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
|
||||
|
||||
let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
|
||||
let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-strict-aws-chunked.zip");
|
||||
let response = signed_put_request_with_headers(
|
||||
&object_url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
zip_bytes.clone(),
|
||||
"application/zip",
|
||||
"aws-chunked",
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let client = env.create_s3_client();
|
||||
assert_archive_object_content_encoding(
|
||||
&client,
|
||||
ARCHIVE_TEST_BUCKET,
|
||||
"bundle-strict-aws-chunked.zip",
|
||||
None,
|
||||
zip_bytes.as_slice(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_archive_put_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
|
||||
env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
|
||||
|
||||
let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
|
||||
let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-strict-aws-chunked-gzip.zip");
|
||||
let response = signed_put_request_with_headers(
|
||||
&object_url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
zip_bytes,
|
||||
"application/zip",
|
||||
"aws-chunked,gzip",
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = response.text().await?;
|
||||
assert!(
|
||||
body.contains("InvalidArgument") || body.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING"),
|
||||
"unexpected error body: {body}"
|
||||
);
|
||||
|
||||
@@ -326,4 +682,258 @@ mod tests {
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_multipart_get_ignores_empty_conditional_etag_headers() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let key = "multipart-empty-conditional-headers.zip";
|
||||
let zip_bytes =
|
||||
complete_archive_multipart_upload_with_content_encoding(&client, MULTIPART_ARCHIVE_TEST_BUCKET, key, None).await?;
|
||||
let object_url = format!("{}/{}/{}", env.url, MULTIPART_ARCHIVE_TEST_BUCKET, key);
|
||||
|
||||
let response = signed_get_request_with_headers(
|
||||
&object_url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
&[("if-match", ""), ("if-none-match", "")],
|
||||
)
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let body = response.bytes().await?;
|
||||
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::OK,
|
||||
"unexpected multipart GET status {status}, body: {}",
|
||||
String::from_utf8_lossy(body.as_ref())
|
||||
);
|
||||
assert_eq!(body.as_ref(), zip_bytes.as_slice());
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_server_with_env(&mut env, &[]).await?;
|
||||
env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let zip_bytes = complete_archive_multipart_upload_with_content_encoding(
|
||||
&client,
|
||||
MULTIPART_ARCHIVE_TEST_BUCKET,
|
||||
"multipart-aws-chunked-gzip.zip",
|
||||
Some("aws-chunked,gzip"),
|
||||
)
|
||||
.await?;
|
||||
assert_archive_object_content_encoding(
|
||||
&client,
|
||||
MULTIPART_ARCHIVE_TEST_BUCKET,
|
||||
"multipart-aws-chunked-gzip.zip",
|
||||
Some("gzip"),
|
||||
zip_bytes.as_slice(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_archive_multipart_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
|
||||
env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let zip_bytes = complete_archive_multipart_upload_with_content_encoding(
|
||||
&client,
|
||||
MULTIPART_ARCHIVE_TEST_BUCKET,
|
||||
"multipart-strict-aws-chunked.zip",
|
||||
Some("aws-chunked"),
|
||||
)
|
||||
.await?;
|
||||
assert_archive_object_content_encoding(
|
||||
&client,
|
||||
MULTIPART_ARCHIVE_TEST_BUCKET,
|
||||
"multipart-strict-aws-chunked.zip",
|
||||
None,
|
||||
zip_bytes.as_slice(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
|
||||
env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let create_result = client
|
||||
.create_multipart_upload()
|
||||
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
|
||||
.key("multipart-strict-aws-chunked-gzip.zip")
|
||||
.content_type("application/zip")
|
||||
.content_encoding("aws-chunked,gzip")
|
||||
.send()
|
||||
.await;
|
||||
let err = create_result.expect_err("strict mode should reject effective archive content encoding");
|
||||
assert_eq!(err.code(), Some("InvalidArgument"));
|
||||
assert!(
|
||||
err.message().is_some_and(|message| {
|
||||
message.contains("Content-Encoding") && message.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING")
|
||||
}),
|
||||
"unexpected error metadata: code={:?}, message={:?}",
|
||||
err.code(),
|
||||
err.message()
|
||||
);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let payload = random_bytes(MULTIPART_PART_SIZE + 768 * 1024);
|
||||
let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?;
|
||||
assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold");
|
||||
|
||||
let create_output = client
|
||||
.create_multipart_upload()
|
||||
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
|
||||
.key("presigned-multipart-bundle.zip")
|
||||
.content_type("application/zip")
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = create_output.upload_id().expect("multipart upload id");
|
||||
|
||||
let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec();
|
||||
let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec();
|
||||
|
||||
let upload_part_1 = client
|
||||
.upload_part()
|
||||
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
|
||||
.key("presigned-multipart-bundle.zip")
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(first_part))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let upload_part_2 = client
|
||||
.upload_part()
|
||||
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
|
||||
.key("presigned-multipart-bundle.zip")
|
||||
.upload_id(upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(second_part))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let completed_upload = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(upload_part_1.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(upload_part_2.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
|
||||
.key("presigned-multipart-bundle.zip")
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed_upload)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let object_url = format!("{}/{}/{}", env.url, MULTIPART_ARCHIVE_TEST_BUCKET, "presigned-multipart-bundle.zip");
|
||||
let direct_response =
|
||||
presigned_get_request_with_accept_encoding(&object_url, &env.access_key, &env.secret_key, "identity").await?;
|
||||
assert_eq!(direct_response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
direct_response
|
||||
.headers()
|
||||
.get("content-length")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<usize>().ok()),
|
||||
Some(zip_bytes.len())
|
||||
);
|
||||
let direct_body = direct_response.bytes().await?;
|
||||
assert_eq!(direct_body.len(), zip_bytes.len());
|
||||
assert_eq!(direct_body.as_ref(), zip_bytes.as_slice());
|
||||
|
||||
let signed = pre_sign_v4(
|
||||
http::Request::builder()
|
||||
.method(http::Method::GET)
|
||||
.uri(object_url.parse::<http::Uri>()?)
|
||||
.header(
|
||||
HOST,
|
||||
object_url
|
||||
.parse::<http::Uri>()?
|
||||
.authority()
|
||||
.ok_or("request URL missing authority")?
|
||||
.to_string(),
|
||||
)
|
||||
.body(Body::empty())?,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"",
|
||||
"us-east-1",
|
||||
600,
|
||||
OffsetDateTime::now_utc(),
|
||||
);
|
||||
let (proxy_url, proxy_handle) = spawn_reverse_proxy_to_presigned_url(signed.uri().to_string()).await?;
|
||||
let proxied_response: reqwest::Response = local_http_client().get(&proxy_url).send().await?;
|
||||
assert_eq!(proxied_response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
proxied_response
|
||||
.headers()
|
||||
.get("content-length")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<usize>().ok()),
|
||||
Some(zip_bytes.len())
|
||||
);
|
||||
let proxied_body: bytes::Bytes = proxied_response.bytes().await?;
|
||||
assert_eq!(proxied_body.len(), zip_bytes.len());
|
||||
assert_eq!(proxied_body.as_ref(), zip_bytes.as_slice());
|
||||
|
||||
proxy_handle.await??;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,441 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{Delete, ObjectIdentifier};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use bytes::Bytes;
|
||||
use clap::Parser;
|
||||
use serde::Serialize;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "small_put_bench")]
|
||||
#[command(about = "Rust-native small PUT benchmark for RustFS-compatible S3 endpoints")]
|
||||
struct Args {
|
||||
#[arg(long, env = "RUSTFS_BENCH_ENDPOINT")]
|
||||
endpoint: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_ACCESS_KEY", default_value = "rustfsadmin")]
|
||||
access_key: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_SECRET_KEY", default_value = "rustfsadmin")]
|
||||
secret_key: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_REGION", default_value = "us-east-1")]
|
||||
region: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_BUCKET", default_value = "small-put-benchmark")]
|
||||
bucket: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_SIZES", default_value = "4KiB,16KiB,64KiB,256KiB,1MiB")]
|
||||
sizes: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_CONCURRENCY", default_value_t = 8)]
|
||||
concurrency: usize,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_DURATION_SECS", default_value_t = 10)]
|
||||
duration_secs: u64,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_TIMEOUT_SECS", default_value_t = 15)]
|
||||
timeout_secs: u64,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_PREFIX")]
|
||||
prefix: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
output_json: Option<PathBuf>,
|
||||
|
||||
#[arg(long, default_value_t = false)]
|
||||
cleanup: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct SizeSpec {
|
||||
label: String,
|
||||
slug: String,
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Sample {
|
||||
ok: bool,
|
||||
duration_ms: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct SizeSummary {
|
||||
label: String,
|
||||
bytes: usize,
|
||||
total: usize,
|
||||
succeeded: usize,
|
||||
failed: usize,
|
||||
wall_secs: f64,
|
||||
object_rate: f64,
|
||||
throughput_mib_per_sec: f64,
|
||||
avg_ms: Option<f64>,
|
||||
p50_ms: Option<f64>,
|
||||
p90_ms: Option<f64>,
|
||||
p99_ms: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RunSummary {
|
||||
run_id: String,
|
||||
endpoint: String,
|
||||
bucket: String,
|
||||
concurrency: usize,
|
||||
duration_secs: u64,
|
||||
timeout_secs: u64,
|
||||
sizes: Vec<SizeSummary>,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.context("failed to build tokio runtime")?;
|
||||
runtime.block_on(async_main())
|
||||
}
|
||||
|
||||
async fn async_main() -> Result<()> {
|
||||
let args = Args::parse();
|
||||
validate_args(&args)?;
|
||||
|
||||
let sizes = parse_size_list(&args.sizes)?;
|
||||
let run_id = args.prefix.clone().unwrap_or_else(default_run_id);
|
||||
let client = build_s3_client(&args.endpoint, &args.access_key, &args.secret_key, &args.region);
|
||||
|
||||
ensure_bucket(&client, &args.bucket).await?;
|
||||
|
||||
let mut size_summaries = Vec::with_capacity(sizes.len());
|
||||
for size in &sizes {
|
||||
let summary = run_size_benchmark(
|
||||
client.clone(),
|
||||
args.bucket.clone(),
|
||||
run_id.clone(),
|
||||
size.clone(),
|
||||
args.concurrency,
|
||||
Duration::from_secs(args.duration_secs),
|
||||
Duration::from_secs(args.timeout_secs),
|
||||
)
|
||||
.await?;
|
||||
print_size_summary(&summary);
|
||||
size_summaries.push(summary);
|
||||
}
|
||||
|
||||
if args.cleanup {
|
||||
cleanup_prefix(&client, &args.bucket, &run_id).await?;
|
||||
}
|
||||
|
||||
let summary = RunSummary {
|
||||
run_id,
|
||||
endpoint: args.endpoint,
|
||||
bucket: args.bucket,
|
||||
concurrency: args.concurrency,
|
||||
duration_secs: args.duration_secs,
|
||||
timeout_secs: args.timeout_secs,
|
||||
sizes: size_summaries,
|
||||
};
|
||||
|
||||
if let Some(path) = args.output_json {
|
||||
let json = serde_json::to_vec_pretty(&summary).context("failed to serialize benchmark summary")?;
|
||||
std::fs::write(&path, json).with_context(|| format!("failed to write benchmark summary to {}", path.display()))?;
|
||||
println!("Wrote summary to {}", path.display());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_args(args: &Args) -> Result<()> {
|
||||
if args.concurrency == 0 {
|
||||
bail!("--concurrency must be greater than zero");
|
||||
}
|
||||
if args.duration_secs == 0 {
|
||||
bail!("--duration-secs must be greater than zero");
|
||||
}
|
||||
if args.timeout_secs == 0 {
|
||||
bail!("--timeout-secs must be greater than zero");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn build_s3_client(endpoint: &str, access_key: &str, secret_key: &str, region: &str) -> Client {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "small-put-bench");
|
||||
let mut config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new(region.to_string()))
|
||||
.endpoint_url(endpoint)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest();
|
||||
|
||||
if endpoint.starts_with("http://") {
|
||||
config = config.http_client(SmithyHttpClientBuilder::new().build_http());
|
||||
}
|
||||
|
||||
Client::from_conf(config.build())
|
||||
}
|
||||
|
||||
async fn ensure_bucket(client: &Client, bucket: &str) -> Result<()> {
|
||||
if client.head_bucket().bucket(bucket).send().await.is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match client.create_bucket().bucket(bucket).send().await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
let rendered = err.to_string();
|
||||
if rendered.contains("BucketAlreadyOwnedByYou") || rendered.contains("BucketAlreadyExists") {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(err).with_context(|| format!("failed to create benchmark bucket {bucket}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_size_benchmark(
|
||||
client: Client,
|
||||
bucket: String,
|
||||
run_id: String,
|
||||
size: SizeSpec,
|
||||
concurrency: usize,
|
||||
duration: Duration,
|
||||
timeout: Duration,
|
||||
) -> Result<SizeSummary> {
|
||||
let payload = Bytes::from(vec![0_u8; size.bytes]);
|
||||
let deadline = Instant::now() + duration;
|
||||
let wall_start = Instant::now();
|
||||
|
||||
let mut handles = Vec::with_capacity(concurrency);
|
||||
for worker in 0..concurrency {
|
||||
let client = client.clone();
|
||||
let bucket = bucket.clone();
|
||||
let payload = payload.clone();
|
||||
let prefix = format!("{run_id}/{}/worker-{worker}", size.slug);
|
||||
handles.push(tokio::spawn(async move {
|
||||
let mut samples = Vec::new();
|
||||
let mut idx = 0usize;
|
||||
|
||||
while Instant::now() < deadline {
|
||||
let key = format!("{prefix}/obj-{idx}.bin");
|
||||
let started_at = Instant::now();
|
||||
let request = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload.clone()))
|
||||
.content_type("application/octet-stream");
|
||||
|
||||
let ok = matches!(tokio::time::timeout(timeout, request.send()).await, Ok(Ok(_)));
|
||||
samples.push(Sample {
|
||||
ok,
|
||||
duration_ms: started_at.elapsed().as_secs_f64() * 1000.0,
|
||||
});
|
||||
idx += 1;
|
||||
}
|
||||
|
||||
samples
|
||||
}));
|
||||
}
|
||||
|
||||
let mut samples = Vec::new();
|
||||
for handle in handles {
|
||||
samples.extend(handle.await.map_err(|err| anyhow!("benchmark worker join error: {err}"))?);
|
||||
}
|
||||
|
||||
Ok(build_size_summary(&size, samples, wall_start.elapsed()))
|
||||
}
|
||||
|
||||
fn build_size_summary(size: &SizeSpec, mut samples: Vec<Sample>, wall_elapsed: Duration) -> SizeSummary {
|
||||
let total = samples.len();
|
||||
let succeeded = samples.iter().filter(|sample| sample.ok).count();
|
||||
let failed = total.saturating_sub(succeeded);
|
||||
let wall_secs = wall_elapsed.as_secs_f64();
|
||||
let object_rate = if wall_secs > 0.0 { succeeded as f64 / wall_secs } else { 0.0 };
|
||||
let throughput_mib_per_sec = if wall_secs > 0.0 {
|
||||
((size.bytes * succeeded) as f64 / (1024.0 * 1024.0)) / wall_secs
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let avg_ms = if total > 0 {
|
||||
Some(samples.iter().map(|sample| sample.duration_ms).sum::<f64>() / total as f64)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
samples.sort_by(|lhs, rhs| lhs.duration_ms.total_cmp(&rhs.duration_ms));
|
||||
let durations: Vec<f64> = samples.into_iter().map(|sample| sample.duration_ms).collect();
|
||||
|
||||
SizeSummary {
|
||||
label: size.label.clone(),
|
||||
bytes: size.bytes,
|
||||
total,
|
||||
succeeded,
|
||||
failed,
|
||||
wall_secs,
|
||||
object_rate,
|
||||
throughput_mib_per_sec,
|
||||
avg_ms,
|
||||
p50_ms: percentile(&durations, 0.50),
|
||||
p90_ms: percentile(&durations, 0.90),
|
||||
p99_ms: percentile(&durations, 0.99),
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_prefix(client: &Client, bucket: &str, prefix: &str) -> Result<()> {
|
||||
let mut continuation_token = None;
|
||||
loop {
|
||||
let response = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.prefix(prefix)
|
||||
.set_continuation_token(continuation_token.clone())
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("failed to list objects for cleanup under {bucket}/{prefix}"))?;
|
||||
|
||||
let objects: Vec<ObjectIdentifier> = response
|
||||
.contents
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(|object| object.key.map(|key| ObjectIdentifier::builder().key(key).build().ok()))
|
||||
.flatten()
|
||||
.collect();
|
||||
|
||||
for chunk in objects.chunks(1_000) {
|
||||
if chunk.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.set_objects(Some(chunk.to_vec()))
|
||||
.quiet(true)
|
||||
.build()
|
||||
.context("failed to build delete request")?,
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("failed to delete cleanup batch under {bucket}/{prefix}"))?;
|
||||
}
|
||||
|
||||
if response.is_truncated.unwrap_or(false) {
|
||||
continuation_token = response.next_continuation_token;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_size_list(input: &str) -> Result<Vec<SizeSpec>> {
|
||||
input
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|item| !item.is_empty())
|
||||
.map(parse_size_spec)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse_size_spec(input: &str) -> Result<SizeSpec> {
|
||||
let normalized = input.trim();
|
||||
let lower = normalized.to_ascii_lowercase();
|
||||
|
||||
let (number_part, multiplier) = if let Some(value) = lower.strip_suffix("kib") {
|
||||
(value, 1024usize)
|
||||
} else if let Some(value) = lower.strip_suffix("mib") {
|
||||
(value, 1024usize * 1024usize)
|
||||
} else if let Some(value) = lower.strip_suffix('b') {
|
||||
(value, 1usize)
|
||||
} else {
|
||||
(lower.as_str(), 1usize)
|
||||
};
|
||||
|
||||
let value = number_part
|
||||
.trim()
|
||||
.parse::<usize>()
|
||||
.with_context(|| format!("invalid size component: {input}"))?;
|
||||
let bytes = value
|
||||
.checked_mul(multiplier)
|
||||
.ok_or_else(|| anyhow!("size overflow for {input}"))?;
|
||||
|
||||
Ok(SizeSpec {
|
||||
label: normalized.to_string(),
|
||||
slug: normalized
|
||||
.chars()
|
||||
.filter(|ch| ch.is_ascii_alphanumeric())
|
||||
.collect::<String>()
|
||||
.to_ascii_lowercase(),
|
||||
bytes,
|
||||
})
|
||||
}
|
||||
|
||||
fn percentile(values: &[f64], percentile: f64) -> Option<f64> {
|
||||
if values.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let index = ((values.len() - 1) as f64 * percentile).floor() as usize;
|
||||
values.get(index).copied()
|
||||
}
|
||||
|
||||
fn default_run_id() -> String {
|
||||
format!("small-put-bench-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"))
|
||||
}
|
||||
|
||||
fn print_size_summary(summary: &SizeSummary) {
|
||||
println!(
|
||||
"{}: success={} failed={} obj/s={:.3} MiB/s={:.3} avg={:.3?} p50={:.3?} p90={:.3?} p99={:.3?}",
|
||||
summary.label,
|
||||
summary.succeeded,
|
||||
summary.failed,
|
||||
summary.object_rate,
|
||||
summary.throughput_mib_per_sec,
|
||||
summary.avg_ms,
|
||||
summary.p50_ms,
|
||||
summary.p90_ms,
|
||||
summary.p99_ms,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_size_spec_supports_binary_units() {
|
||||
let four_kib = parse_size_spec("4KiB").expect("4KiB should parse");
|
||||
assert_eq!(four_kib.bytes, 4 * 1024);
|
||||
|
||||
let one_mib = parse_size_spec("1MiB").expect("1MiB should parse");
|
||||
assert_eq!(one_mib.bytes, 1024 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percentile_returns_expected_bucket() {
|
||||
let values = vec![10.0, 20.0, 30.0, 40.0, 50.0];
|
||||
assert_eq!(percentile(&values, 0.50), Some(30.0));
|
||||
assert_eq!(percentile(&values, 0.90), Some(40.0));
|
||||
}
|
||||
}
|
||||
@@ -19,13 +19,9 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::{ByteStream, SdkBody};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
|
||||
use base64::Engine;
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use http_body::Frame;
|
||||
use http_body_util::StreamBody;
|
||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -68,53 +64,6 @@ mod tests {
|
||||
.encoded
|
||||
}
|
||||
|
||||
fn streamed_body_70kib_of_a() -> ByteStream {
|
||||
let bytes = Bytes::from_static(&[b'a'; 1024]);
|
||||
let stream = futures::stream::repeat_with(move || {
|
||||
let frame = Frame::data(bytes.clone());
|
||||
Ok::<_, std::io::Error>(frame)
|
||||
});
|
||||
let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024);
|
||||
ByteStream::new(SdkBody::from_body_1_x(body))
|
||||
}
|
||||
|
||||
struct WithSizeHint<T> {
|
||||
inner: T,
|
||||
size_hint: usize,
|
||||
}
|
||||
|
||||
impl<T> WithSizeHint<T> {
|
||||
fn new(inner: T, size_hint: usize) -> Self {
|
||||
Self { inner, size_hint }
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> http_body::Body for WithSizeHint<T>
|
||||
where
|
||||
T: http_body::Body + Unpin,
|
||||
{
|
||||
type Data = T::Data;
|
||||
type Error = T::Error;
|
||||
|
||||
fn poll_frame(
|
||||
self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> std::task::Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
||||
let this = self.get_mut();
|
||||
std::pin::Pin::new(&mut this.inner).poll_frame(cx)
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.is_end_stream()
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
let mut hint = self.inner.size_hint();
|
||||
hint.set_exact(self.size_hint as u64);
|
||||
hint
|
||||
}
|
||||
}
|
||||
|
||||
/// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
@@ -187,121 +136,6 @@ mod tests {
|
||||
info!("PASSED: PutObject with checksum_sha256 and GetObject content match");
|
||||
}
|
||||
|
||||
/// Mirrors `s3s-e2e` behavior: only request `checksum_algorithm`, then expect
|
||||
/// both PutObject and GetObject(checksum_mode=enabled) to expose the same checksum.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_object_with_checksum_algorithm_only() {
|
||||
init_logging();
|
||||
info!("TEST: PutObject with checksum_algorithm only");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-checksum-algorithm-only";
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let key = "obj-with-checksum-algorithm-only.txt";
|
||||
let content = vec![b'a'; 70 * 1024];
|
||||
|
||||
let put_resp = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.body(ByteStream::from(content.clone()))
|
||||
.send()
|
||||
.await
|
||||
.expect("PutObject with checksum_algorithm should succeed");
|
||||
|
||||
let put_checksum = put_resp
|
||||
.checksum_crc32()
|
||||
.expect("PutObject should return checksum_crc32 when checksum_algorithm is used")
|
||||
.to_string();
|
||||
|
||||
let mut get_resp = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("GetObject should succeed");
|
||||
|
||||
let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(aws_sdk_s3::primitives::SdkBody::empty()))
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect body")
|
||||
.into_bytes();
|
||||
|
||||
assert_eq!(body_bytes.as_ref(), content.as_slice(), "GetObject body must match uploaded content");
|
||||
assert_eq!(
|
||||
get_resp.checksum_crc32().map(str::to_string),
|
||||
Some(put_checksum),
|
||||
"GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum"
|
||||
);
|
||||
}
|
||||
|
||||
/// Matches the `s3s-e2e` streaming upload shape more closely than `ByteStream::from(Vec<u8>)`.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_object_with_checksum_algorithm_only_streaming_body() {
|
||||
init_logging();
|
||||
info!("TEST: PutObject with checksum_algorithm only using streaming body");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-checksum-algorithm-streaming";
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let key = "obj-with-checksum-algorithm-streaming.txt";
|
||||
let expected_content = vec![b'a'; 70 * 1024];
|
||||
|
||||
let put_resp = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.body(streamed_body_70kib_of_a())
|
||||
.send()
|
||||
.await
|
||||
.expect("PutObject with streaming checksum_algorithm should succeed");
|
||||
|
||||
let put_checksum = put_resp
|
||||
.checksum_crc32()
|
||||
.expect("PutObject should return checksum_crc32 for streaming checksum_algorithm uploads")
|
||||
.to_string();
|
||||
|
||||
let mut get_resp = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("GetObject should succeed");
|
||||
|
||||
let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(SdkBody::empty()))
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect body")
|
||||
.into_bytes();
|
||||
|
||||
assert_eq!(
|
||||
body_bytes.as_ref(),
|
||||
expected_content.as_slice(),
|
||||
"GetObject body must match uploaded content"
|
||||
);
|
||||
assert_eq!(
|
||||
get_resp.checksum_crc32().map(str::to_string),
|
||||
Some(put_checksum),
|
||||
"GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum for streaming uploads"
|
||||
);
|
||||
}
|
||||
|
||||
/// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
|
||||
/// Uses part size >= 5MB (server minimum) for two parts.
|
||||
#[tokio::test]
|
||||
@@ -400,114 +234,6 @@ mod tests {
|
||||
info!("PASSED: MultipartUpload with checksum and GetObject content match");
|
||||
}
|
||||
|
||||
/// Mirrors `s3s-e2e` multipart behavior: request checksum algorithm at MPU creation,
|
||||
/// rely on auto checksum handling during UploadPart, and expect CompleteMultipartUpload to succeed.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_multipart_upload_with_crc32_algorithm_only() {
|
||||
init_logging();
|
||||
info!("TEST: MultipartUpload with checksum_algorithm only (CRC32)");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-multipart-checksum-crc32-auto";
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let key = "multipart-with-crc32-auto.bin";
|
||||
let part1_content = "a".repeat(5 * 1024 * 1024 + 1);
|
||||
let part2_content = "b".repeat(1024);
|
||||
|
||||
let create_resp = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.send()
|
||||
.await
|
||||
.expect("CreateMultipartUpload should succeed");
|
||||
|
||||
let upload_id = create_resp.upload_id().expect("upload_id should be present");
|
||||
|
||||
let part1_resp = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(part1_content.clone().into_bytes()))
|
||||
.send()
|
||||
.await
|
||||
.expect("UploadPart 1 should succeed");
|
||||
let part1_checksum = part1_resp
|
||||
.checksum_crc32()
|
||||
.expect("UploadPart 1 should return checksum_crc32")
|
||||
.to_string();
|
||||
|
||||
let part2_resp = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(part2_content.clone().into_bytes()))
|
||||
.send()
|
||||
.await
|
||||
.expect("UploadPart 2 should succeed");
|
||||
let part2_checksum = part2_resp
|
||||
.checksum_crc32()
|
||||
.expect("UploadPart 2 should return checksum_crc32")
|
||||
.to_string();
|
||||
|
||||
let completed_upload = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(part1_resp.e_tag().expect("etag part 1"))
|
||||
.checksum_crc32(part1_checksum)
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(part2_resp.e_tag().expect("etag part 2"))
|
||||
.checksum_crc32(part2_checksum)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed_upload)
|
||||
.send()
|
||||
.await
|
||||
.expect("CompleteMultipartUpload should succeed");
|
||||
|
||||
let body_bytes = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect("GetObject should succeed")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect body")
|
||||
.into_bytes();
|
||||
|
||||
let expected_content = format!("{part1_content}{part2_content}");
|
||||
assert_eq!(
|
||||
body_bytes.as_ref(),
|
||||
expected_content.as_bytes(),
|
||||
"completed multipart object must match concatenated parts"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression test for issue #2282:
|
||||
/// CRC64NVME full-object checksum should match between direct PutObject and multipart upload.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -138,4 +138,63 @@ mod tests {
|
||||
|
||||
env.stop_server();
|
||||
}
|
||||
|
||||
/// Issue #2475 / Route A: when aws-chunked is combined with an effective object encoding,
|
||||
/// only the effective encoding should roundtrip through GET/HEAD.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_content_encoding_aws_chunked_with_effective_encoding_roundtrip() {
|
||||
init_logging();
|
||||
info!("aws-chunked,gzip should persist only gzip");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "content-encoding-aws-chunked-gzip-test";
|
||||
let key = "streamed/object.txt";
|
||||
let content = b"streaming upload body with effective gzip encoding";
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to create bucket");
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.content_type("text/plain")
|
||||
.content_encoding("aws-chunked,gzip")
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT failed");
|
||||
|
||||
let get_resp = client.get_object().bucket(bucket).key(key).send().await.expect("GET failed");
|
||||
assert_eq!(
|
||||
get_resp.content_encoding(),
|
||||
Some("gzip"),
|
||||
"GET must return only the effective content encoding after aws-chunked is stripped"
|
||||
);
|
||||
let body = get_resp.body.collect().await.unwrap().into_bytes();
|
||||
assert_eq!(body.as_ref(), content, "Body content mismatch");
|
||||
|
||||
let head_resp = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect("HEAD failed");
|
||||
assert_eq!(
|
||||
head_resp.content_encoding(),
|
||||
Some("gzip"),
|
||||
"HEAD must return only the effective content encoding after aws-chunked is stripped"
|
||||
);
|
||||
|
||||
env.stop_server();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -344,9 +344,6 @@ async fn test_local_kms_multipart_upload() {
|
||||
test_multipart_upload_with_sse_c(&s3_client, TEST_BUCKET)
|
||||
.await
|
||||
.expect("SSE-C multipart upload test failed");
|
||||
test_multipart_download_with_wrong_sse_c_key_fails(&s3_client, TEST_BUCKET)
|
||||
.await
|
||||
.expect("SSE-C multipart wrong-key download test failed");
|
||||
|
||||
// Test 4: Large multipart upload (test streaming encryption with multiple blocks)
|
||||
// TODO: Re-enable after fixing streaming encryption issues with large files
|
||||
@@ -651,99 +648,6 @@ async fn test_multipart_upload_with_sse_c(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn test_multipart_download_with_wrong_sse_c_key_fails(
|
||||
s3_client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let object_key = "multipart-sse-c-bad-download-test";
|
||||
let part_size = 5 * 1024 * 1024;
|
||||
let total_parts = 2;
|
||||
let total_size = part_size * total_parts;
|
||||
|
||||
let encryption_key = "01234567890123456789012345678901";
|
||||
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
|
||||
let key_md5 = sse_customer_key_md5_base64(encryption_key);
|
||||
|
||||
let wrong_key = "abcdefghijklmnopqrstuvwxyz012345";
|
||||
let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
|
||||
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
|
||||
|
||||
let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 5) % 256) as u8).collect();
|
||||
|
||||
let create_multipart_output = s3_client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&key_b64)
|
||||
.sse_customer_key_md5(&key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let upload_id = create_multipart_output.upload_id().unwrap();
|
||||
let mut completed_parts = Vec::new();
|
||||
|
||||
for part_number in 1..=total_parts {
|
||||
let start = (part_number - 1) * part_size;
|
||||
let end = std::cmp::min(start + part_size, total_size);
|
||||
let part_data = &test_data[start..end];
|
||||
|
||||
let upload_part_output = s3_client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(part_number as i32)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec()))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&key_b64)
|
||||
.sse_customer_key_md5(&key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
completed_parts.push(
|
||||
aws_sdk_s3::types::CompletedPart::builder()
|
||||
.part_number(part_number as i32)
|
||||
.e_tag(upload_part_output.e_tag().unwrap())
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
|
||||
let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
|
||||
.set_parts(Some(completed_parts))
|
||||
.build();
|
||||
|
||||
s3_client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed_multipart_upload)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let err = s3_client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&wrong_key_b64)
|
||||
.sse_customer_key_md5(&wrong_key_md5)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("multipart SSE-C download with the wrong key should fail");
|
||||
|
||||
let service_err = err.into_service_error();
|
||||
assert_eq!(
|
||||
service_err.meta().code(),
|
||||
Some("InvalidRequest"),
|
||||
"wrong-key multipart SSE-C download should return InvalidRequest, got {:?}",
|
||||
service_err.meta().code()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test large multipart upload to verify streaming encryption works correctly
|
||||
#[allow(dead_code)]
|
||||
async fn test_large_multipart_upload(
|
||||
|
||||
@@ -450,11 +450,7 @@ impl KMSTestSuite {
|
||||
if failed > 0 {
|
||||
warn!("❌ Failing tests:");
|
||||
for result in results.iter().filter(|r| !r.success) {
|
||||
warn!(
|
||||
" - {}: {}",
|
||||
result.test_name,
|
||||
result.error_message.as_ref().unwrap_or(&"Unknown error".to_string())
|
||||
);
|
||||
warn!(" - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,10 +100,6 @@ mod cluster_concurrency_test;
|
||||
#[cfg(test)]
|
||||
mod checksum_upload_test;
|
||||
|
||||
// Range request regression tests
|
||||
#[cfg(test)]
|
||||
mod range_request_test;
|
||||
|
||||
// Group deletion tests
|
||||
#[cfg(test)]
|
||||
mod group_delete_test;
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use serial_test::serial;
|
||||
use tracing::info;
|
||||
@@ -179,4 +180,84 @@ mod tests {
|
||||
"Delete marker should no longer be latest after the second put"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_list_object_versions_prefix_with_marker_object_returns_children() {
|
||||
init_logging();
|
||||
info!("🧪 TEST: ListObjectVersions returns prefix children when a marker object also exists");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-list-versions-prefix-marker";
|
||||
let marker_key = "data01";
|
||||
let child_keys = [
|
||||
"data01/meta/dump-2026-04-08-053205.json.gz",
|
||||
"data01/meta/dump-2026-04-08-063209.json.gz",
|
||||
];
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to create bucket");
|
||||
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Suspended)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to suspend versioning");
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(marker_key)
|
||||
.body(ByteStream::from_static(b""))
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to put marker object");
|
||||
|
||||
for key in child_keys {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"payload"))
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to put child object");
|
||||
}
|
||||
|
||||
let listing = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix("data01/")
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to list object versions by prefix");
|
||||
|
||||
let version_keys: Vec<_> = listing.versions().iter().filter_map(|version| version.key()).collect();
|
||||
|
||||
assert_eq!(
|
||||
version_keys.len(),
|
||||
child_keys.len(),
|
||||
"ListObjectVersions with a trailing slash prefix should include child objects even when the marker object exists"
|
||||
);
|
||||
|
||||
for key in child_keys {
|
||||
assert!(
|
||||
version_keys.contains(&key),
|
||||
"ListObjectVersions(prefix=data01/) should include child object {key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! End-to-end regression test for invalid GET object ranges.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use serial_test::serial;
|
||||
use tracing::info;
|
||||
|
||||
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
|
||||
env.create_s3_client()
|
||||
}
|
||||
|
||||
async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
match client.create_bucket().bucket(bucket).send().await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
if err.to_string().contains("BucketAlreadyOwnedByYou") || err.to_string().contains("BucketAlreadyExists") {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Box::new(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_object_invalid_range_returns_416_issue_s3_implemented_tests() {
|
||||
init_logging();
|
||||
info!("TEST: GetObject invalid range should return InvalidRange/416");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-invalid-range";
|
||||
let key = "range.txt";
|
||||
let content = b"testcontent";
|
||||
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("PutObject should succeed");
|
||||
|
||||
let result = client.get_object().bucket(bucket).key(key).range("bytes=40-50").send().await;
|
||||
|
||||
let err = result.expect_err("GetObject with an unsatisfiable range should fail");
|
||||
match err {
|
||||
SdkError::ServiceError(service_err) => {
|
||||
assert_eq!(service_err.raw().status().as_u16(), 416, "invalid range should return HTTP 416");
|
||||
|
||||
let s3_err = service_err.into_err();
|
||||
assert_eq!(s3_err.meta().code(), Some("InvalidRange"), "invalid range should map to InvalidRange");
|
||||
}
|
||||
other_err => panic!("Expected S3 service error, got: {other_err:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -259,7 +259,7 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
|
||||
|
||||
let upload_id = initiate_response
|
||||
.upload_id()
|
||||
.ok_or(std::io::Error::other("No upload ID returned"))?;
|
||||
.ok_or_else(|| std::io::Error::other("No upload ID returned"))?;
|
||||
|
||||
// Upload parts
|
||||
for part_number in 1..=num_parts {
|
||||
@@ -277,7 +277,7 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
|
||||
|
||||
let part_etag = upload_part_response
|
||||
.e_tag()
|
||||
.ok_or(std::io::Error::other("Do not have etag"))?
|
||||
.ok_or_else(|| std::io::Error::other("Do not have etag"))?
|
||||
.to_string();
|
||||
|
||||
let completed_part = CompletedPart::builder().part_number(part_number).e_tag(part_etag).build();
|
||||
|
||||
@@ -120,11 +120,6 @@ impl LockClient for GrpcLockClient {
|
||||
.map_err(|e| LockError::internal(e.to_string()))?
|
||||
.into_inner();
|
||||
|
||||
// Check for explicit error first
|
||||
if let Some(error_info) = resp.error_info {
|
||||
return Err(LockError::internal(error_info));
|
||||
}
|
||||
|
||||
// Check if the lock acquisition was successful
|
||||
if resp.success {
|
||||
Ok(LockResponse::success(
|
||||
@@ -134,7 +129,8 @@ impl LockClient for GrpcLockClient {
|
||||
} else {
|
||||
// Lock acquisition failed
|
||||
Ok(LockResponse::failure(
|
||||
"Lock acquisition failed on remote server".to_string(),
|
||||
resp.error_info
|
||||
.unwrap_or_else(|| "Lock acquisition failed on remote server".to_string()),
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -50,6 +50,18 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> GenerallyLockResult {
|
||||
if success {
|
||||
GenerallyLockResult {
|
||||
success: true,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
}
|
||||
} else {
|
||||
lock_result_from_error(format!("lock not found for release: {lock_id}"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal NodeService implementation that only supports Lock RPCs
|
||||
/// Used for testing distributed lock scenarios with real gRPC
|
||||
#[derive(Debug)]
|
||||
@@ -102,7 +114,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
let lock_info_json = result.lock_info.as_ref().and_then(|info| serde_json::to_string(info).ok());
|
||||
Ok(Response::new(GenerallyLockResponse {
|
||||
success: result.success,
|
||||
error_info: None,
|
||||
error_info: result.error,
|
||||
lock_info: lock_info_json,
|
||||
}))
|
||||
}
|
||||
@@ -131,11 +143,14 @@ impl NodeService for MinimalLockNodeService {
|
||||
};
|
||||
|
||||
match self.lock_client.release(&args.lock_id).await {
|
||||
Ok(success) => Ok(Response::new(GenerallyLockResponse {
|
||||
success,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
})),
|
||||
Ok(success) => {
|
||||
let result = lock_result_from_release(&args.lock_id, success);
|
||||
Ok(Response::new(GenerallyLockResponse {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(format!(
|
||||
@@ -161,11 +176,14 @@ impl NodeService for MinimalLockNodeService {
|
||||
};
|
||||
|
||||
match self.lock_client.force_release(&args.lock_id).await {
|
||||
Ok(success) => Ok(Response::new(GenerallyLockResponse {
|
||||
success,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
})),
|
||||
Ok(success) => {
|
||||
let result = lock_result_from_release(&args.lock_id, success);
|
||||
Ok(Response::new(GenerallyLockResponse {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(format!(
|
||||
@@ -271,10 +289,9 @@ impl NodeService for MinimalLockNodeService {
|
||||
Ok(batch_results) => {
|
||||
for (result_idx, success) in batch_results.into_iter().enumerate() {
|
||||
if let Some(request_idx) = valid_indices.get(result_idx) {
|
||||
results[*request_idx] = GenerallyLockResult {
|
||||
success,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
results[*request_idx] = match lock_ids.get(result_idx) {
|
||||
Some(lock_id) => lock_result_from_release(lock_id, success),
|
||||
None => lock_result_from_error(format!("unlock response index out of range: {result_idx}")),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,7 +160,7 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error:
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "requires running RustFS server at localhost:9000"]
|
||||
async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::error::Error>> {
|
||||
async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::error::Error>> {
|
||||
use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
|
||||
|
||||
let client = create_aws_s3_client().await?;
|
||||
@@ -176,19 +176,12 @@ async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::er
|
||||
.build()?;
|
||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||
|
||||
let err = client
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(BUCKET)
|
||||
.lifecycle_configuration(lifecycle)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("zero-day lifecycle expiration should be rejected");
|
||||
|
||||
let err_msg = format!("{err:?}");
|
||||
assert!(
|
||||
err_msg.contains("InvalidArgument") && err_msg.contains("greater than 0"),
|
||||
"unexpected error: {err_msg}"
|
||||
);
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -275,6 +275,41 @@ async fn test_grpc_lock_client_batch_acquire_and_release() {
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_grpc_lock_client_uses_request_lock_id_and_reports_missing_unlock() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let local_client: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager));
|
||||
|
||||
let (addr, handle) = spawn_lock_server(local_client).await.expect("Failed to spawn server");
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let grpc_client = GrpcLockClient::new(addr);
|
||||
let request = LockRequest::new(test_resource(), LockType::Exclusive, "owner-a").with_acquire_timeout(Duration::from_secs(2));
|
||||
|
||||
let response = grpc_client.acquire_lock(&request).await.expect("gRPC acquire should succeed");
|
||||
let lock_info = response.lock_info.expect("gRPC acquire should include lock info");
|
||||
assert_eq!(lock_info.id, request.lock_id);
|
||||
|
||||
assert!(
|
||||
grpc_client
|
||||
.release(&request.lock_id)
|
||||
.await
|
||||
.expect("gRPC release should succeed"),
|
||||
"release should find the request lock id"
|
||||
);
|
||||
|
||||
let missing_release = grpc_client
|
||||
.release(&request.lock_id)
|
||||
.await
|
||||
.expect_err("second release should report missing lock");
|
||||
assert!(
|
||||
missing_release.to_string().contains("lock not found for release"),
|
||||
"missing release should preserve server error, got: {missing_release}"
|
||||
);
|
||||
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_distributed_lock_4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes() {
|
||||
let manager1 = Arc::new(GlobalLockManager::new());
|
||||
|
||||
@@ -154,7 +154,7 @@ async fn walk_dir() -> Result<(), Box<dyn Error>> {
|
||||
match response.next().await {
|
||||
Some(Ok(resp)) => {
|
||||
if !resp.success {
|
||||
println!("{}", resp.error_info.unwrap_or("".to_string()));
|
||||
println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
|
||||
}
|
||||
let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
|
||||
.map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))
|
||||
|
||||
@@ -281,8 +281,11 @@ async fn test_select_object_content_csv_limit() -> Result<(), Box<dyn Error>> {
|
||||
println!("CSV Limit result: {result_str}");
|
||||
|
||||
// Verify only first 2 records are returned
|
||||
let lines: Vec<&str> = result_str.lines().filter(|line| !line.trim().is_empty()).collect();
|
||||
assert_eq!(lines.len(), 2, "Should return exactly 2 records");
|
||||
assert_eq!(
|
||||
result_str.lines().filter(|line| !line.trim().is_empty()).count(),
|
||||
2,
|
||||
"Should return exactly 2 records"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -321,8 +324,10 @@ async fn test_select_object_content_csv_order_by() -> Result<(), Box<dyn Error>>
|
||||
println!("CSV Order By result: {result_str}");
|
||||
|
||||
// Verify ordered by age descending
|
||||
let lines: Vec<&str> = result_str.lines().filter(|line| !line.trim().is_empty()).collect();
|
||||
assert!(lines.len() >= 2, "Should return at least 2 records");
|
||||
assert!(
|
||||
result_str.lines().filter(|line| !line.trim().is_empty()).count() >= 2,
|
||||
"Should return at least 2 records"
|
||||
);
|
||||
|
||||
// Check if contains highest age records
|
||||
assert!(result_str.contains("Charlie,35"));
|
||||
|
||||
@@ -190,6 +190,14 @@ async fn put_bucket_replication(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_bucket_replication(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
|
||||
let url = format!("{}/{bucket}?replication", env.url);
|
||||
signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await
|
||||
}
|
||||
|
||||
async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
@@ -1084,6 +1092,54 @@ async fn test_remove_remote_target_rejects_target_used_by_replication() -> Resul
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_bucket_replication_removes_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
source_env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = "replication-delete-config-src";
|
||||
let target_bucket = "replication-delete-config-dst";
|
||||
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let delete_response = delete_bucket_replication(&source_env, source_bucket).await?;
|
||||
assert!(
|
||||
delete_response.status().is_success(),
|
||||
"unexpected delete status: {}",
|
||||
delete_response.status()
|
||||
);
|
||||
|
||||
let targets_response = list_replication_targets_request(&source_env, Some(source_bucket)).await?;
|
||||
assert_eq!(targets_response.status(), StatusCode::OK);
|
||||
let targets: Vec<serde_json::Value> = targets_response.json().await?;
|
||||
assert!(
|
||||
targets
|
||||
.iter()
|
||||
.all(|target| target.get("arn").and_then(|arn| arn.as_str()) != Some(target_arn.as_str())),
|
||||
"deleted replication config left stale target {target_arn}: {targets:?}"
|
||||
);
|
||||
|
||||
let recreated_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &recreated_arn).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
|
||||
@@ -26,10 +26,16 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::StatusCode;
|
||||
use http::header::HOST;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Helper function to create an S3 client for testing
|
||||
@@ -56,6 +62,30 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn signed_get(
|
||||
url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
|
||||
let request = http::Request::builder()
|
||||
.method(http::Method::GET)
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
|
||||
.body(Body::empty())?;
|
||||
|
||||
let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
|
||||
let client = local_http_client();
|
||||
let mut request_builder = client.get(url);
|
||||
for (name, value) in signed.headers() {
|
||||
request_builder = request_builder.header(name, value);
|
||||
}
|
||||
|
||||
Ok(request_builder.send().await?)
|
||||
}
|
||||
|
||||
/// Test PUT and GET with space character in path
|
||||
///
|
||||
/// This reproduces Part A of the issue:
|
||||
@@ -274,6 +304,73 @@ mod tests {
|
||||
info!("Test completed successfully");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_get_missing_object_with_trailing_equals_returns_no_such_key() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-missing-equals-key";
|
||||
create_bucket(&client, bucket).await?;
|
||||
|
||||
let url = format!("{}/{}/path/sitemap.xmlage=", env.url, bucket);
|
||||
let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
|
||||
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::NOT_FOUND,
|
||||
"missing object key ending with '=' should pass signature validation before object lookup"
|
||||
);
|
||||
|
||||
let body = response.text().await?;
|
||||
assert!(body.contains("<Code>NoSuchKey</Code>"), "expected NoSuchKey XML response, got: {body}");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_get_existing_object_with_trailing_equals_returns_content() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-existing-equals-key";
|
||||
create_bucket(&client, bucket).await?;
|
||||
|
||||
let key = "path/sitemap.xmlage=";
|
||||
let content = b"object content for raw signed URL with trailing equals";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let url = format!("{}/{}/{}", env.url, bucket, key);
|
||||
let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
|
||||
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::OK,
|
||||
"existing object key ending with '=' should pass signature validation and return content"
|
||||
);
|
||||
|
||||
let body = response.bytes().await?;
|
||||
assert_eq!(body.as_ref(), content);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test DELETE operation with special characters
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
|
||||
@@ -70,7 +70,6 @@ reed-solomon-erasure = { workspace = true }
|
||||
reed-solomon-simd = { workspace = true }
|
||||
lazy_static.workspace = true
|
||||
rustfs-lock.workspace = true
|
||||
rustfs-io-core.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
regex = { workspace = true }
|
||||
path-absolutize = { workspace = true }
|
||||
@@ -87,7 +86,7 @@ hyper.workspace = true
|
||||
hyper-util.workspace = true
|
||||
hyper-rustls.workspace = true
|
||||
rustls.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
|
||||
tokio = { workspace = true, features = ["io-util", "sync", "signal","io-uring"] }
|
||||
tonic.workspace = true
|
||||
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
|
||||
tower.workspace = true
|
||||
@@ -140,17 +139,5 @@ harness = false
|
||||
name = "comparison_benchmark"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "direct_chunk_benchmark"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "reconstructed_chunk_benchmark"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "bitrot_chunk_benchmark"
|
||||
harness = false
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -32,43 +32,6 @@
|
||||
|
||||
For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
|
||||
|
||||
## 📈 Benchmarks
|
||||
|
||||
ECStore ships several Criterion benchmarks under [`crates/ecstore/benches/`](./benches/).
|
||||
|
||||
### Direct Chunk Path
|
||||
|
||||
Use the direct chunk benchmark to compare the current slice-forwarding path against the previous assembled-copy path:
|
||||
|
||||
```bash
|
||||
cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark
|
||||
```
|
||||
|
||||
To run only the end-to-end ECStore range-read benchmark:
|
||||
|
||||
```bash
|
||||
cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark ecstore_get_object_chunks
|
||||
```
|
||||
|
||||
To run the reconstructed multi-disk range-read benchmark:
|
||||
|
||||
```bash
|
||||
cargo bench -p rustfs-ecstore --bench reconstructed_chunk_benchmark
|
||||
```
|
||||
|
||||
### Saved Comparison Points
|
||||
|
||||
Latest local measurements on this branch:
|
||||
|
||||
- `direct_chunk_path/slice_forwarding/single_block_aligned`: about `477 ns`
|
||||
- `direct_chunk_path/assembled_copy/single_block_aligned`: about `3.25 us`
|
||||
- `direct_chunk_path/slice_forwarding/multi_block_unaligned`: about `963 ns`
|
||||
- `direct_chunk_path/assembled_copy/multi_block_unaligned`: about `7.25 us`
|
||||
- `ecstore_get_object_chunks/drain/multi_disk_range`: about `644-654 us`, throughput about `2.86-2.90 GiB/s`
|
||||
- `reconstructed_chunk_path/drain/multi_disk_missing_shard`: about `1.292-1.304 ms`, throughput about `1.43-1.45 GiB/s`
|
||||
|
||||
These numbers are intended as branch-local reference points. Re-run the benchmark on your target machine before treating them as a regression baseline.
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use bytes::Bytes;
|
||||
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
|
||||
use rustfs_ecstore::bitrot::decode_bitrot_chunk_source_for_bench;
|
||||
use rustfs_io_core::IoChunk;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use std::hint::black_box;
|
||||
|
||||
struct BitrotChunkBenchCase {
|
||||
name: &'static str,
|
||||
source_chunks: Vec<IoChunk>,
|
||||
shard_size: usize,
|
||||
expected_decoded_len: usize,
|
||||
expected_copied: bool,
|
||||
}
|
||||
|
||||
fn encode_shard(checksum_algo: HashAlgorithm, shard: &[u8]) -> Vec<u8> {
|
||||
let mut encoded = Vec::with_capacity(checksum_algo.size() + shard.len());
|
||||
encoded.extend_from_slice(checksum_algo.hash_encode(shard).as_ref());
|
||||
encoded.extend_from_slice(shard);
|
||||
encoded
|
||||
}
|
||||
|
||||
fn bitrot_chunk_bench_cases() -> [BitrotChunkBenchCase; 2] {
|
||||
let checksum_algo = HashAlgorithm::Md5;
|
||||
let shard_one = b"abcd";
|
||||
let shard_two = b"efgh";
|
||||
let encoded_one = encode_shard(checksum_algo.clone(), shard_one);
|
||||
let encoded_two = encode_shard(checksum_algo.clone(), shard_two);
|
||||
|
||||
let mut cross_chunk = Vec::with_capacity(encoded_one.len() + encoded_two.len());
|
||||
cross_chunk.extend_from_slice(&encoded_one);
|
||||
cross_chunk.extend_from_slice(&encoded_two);
|
||||
let split = checksum_algo.size() + 2;
|
||||
|
||||
[
|
||||
BitrotChunkBenchCase {
|
||||
name: "aligned_multi_chunk_no_copy",
|
||||
source_chunks: vec![
|
||||
IoChunk::Shared(Bytes::from(encoded_one)),
|
||||
IoChunk::Shared(Bytes::from(encoded_two)),
|
||||
],
|
||||
shard_size: shard_one.len(),
|
||||
expected_decoded_len: shard_one.len() + shard_two.len(),
|
||||
expected_copied: false,
|
||||
},
|
||||
BitrotChunkBenchCase {
|
||||
name: "cross_chunk_frame_copy",
|
||||
source_chunks: vec![
|
||||
IoChunk::Shared(Bytes::copy_from_slice(&cross_chunk[..split])),
|
||||
IoChunk::Shared(Bytes::copy_from_slice(&cross_chunk[split..])),
|
||||
],
|
||||
shard_size: shard_one.len(),
|
||||
expected_decoded_len: shard_one.len() + shard_two.len(),
|
||||
expected_copied: true,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn bench_bitrot_chunk_decode(c: &mut Criterion) {
|
||||
let checksum_algo = HashAlgorithm::Md5;
|
||||
let mut group = c.benchmark_group("bitrot_chunk_decode");
|
||||
group.sample_size(20);
|
||||
|
||||
for case in bitrot_chunk_bench_cases() {
|
||||
let (decoded, copied) =
|
||||
decode_bitrot_chunk_source_for_bench(&case.source_chunks, case.shard_size, checksum_algo.clone(), false)
|
||||
.expect("decode bitrot source");
|
||||
let decoded_len: usize = decoded.iter().map(IoChunk::len).sum();
|
||||
|
||||
assert_eq!(decoded_len, case.expected_decoded_len);
|
||||
assert_eq!(copied, case.expected_copied);
|
||||
|
||||
group.throughput(Throughput::Bytes(case.expected_decoded_len as u64));
|
||||
group.bench_with_input(BenchmarkId::new("decode", case.name), &case, |b, case| {
|
||||
b.iter(|| {
|
||||
let result = decode_bitrot_chunk_source_for_bench(
|
||||
black_box(&case.source_chunks),
|
||||
black_box(case.shard_size),
|
||||
checksum_algo.clone(),
|
||||
false,
|
||||
)
|
||||
.expect("decode bitrot source");
|
||||
black_box(result);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_bitrot_chunk_decode);
|
||||
criterion_main!(benches);
|
||||
@@ -1,390 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
|
||||
use futures_util::StreamExt;
|
||||
use futures_util::stream;
|
||||
use http::HeaderMap;
|
||||
use rustfs_ecstore::bucket::metadata_sys;
|
||||
use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||
use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use rustfs_ecstore::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
|
||||
use rustfs_ecstore::set_disk::collect_direct_data_shard_chunks_for_benchmark;
|
||||
use rustfs_ecstore::store::{ECStore, init_local_disks};
|
||||
use rustfs_ecstore::store_api::{
|
||||
BucketOperations, BucketOptions, ChunkNativePutData, GetObjectChunkCopyMode, HTTPRangeSpec, MakeBucketOptions, ObjectIO,
|
||||
ObjectOptions,
|
||||
};
|
||||
use rustfs_io_core::{BoxChunkStream, IoChunk, MappedChunk};
|
||||
use std::hint::black_box;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU16, Ordering};
|
||||
use tempfile::TempDir;
|
||||
use tokio::runtime::Runtime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct BenchCase {
|
||||
name: &'static str,
|
||||
data_shards: usize,
|
||||
block_size: usize,
|
||||
blocks: usize,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
}
|
||||
|
||||
fn bench_cases() -> [BenchCase; 2] {
|
||||
[
|
||||
BenchCase {
|
||||
name: "single_block_aligned",
|
||||
data_shards: 4,
|
||||
block_size: 256 * 1024,
|
||||
blocks: 1,
|
||||
offset: 0,
|
||||
length: 256 * 1024,
|
||||
},
|
||||
BenchCase {
|
||||
name: "multi_block_unaligned",
|
||||
data_shards: 4,
|
||||
block_size: 256 * 1024,
|
||||
blocks: 8,
|
||||
offset: 123_457,
|
||||
length: 2 * 256 * 1024 + 33_333,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct EcstoreBenchCase {
|
||||
name: &'static str,
|
||||
disk_count: usize,
|
||||
payload_len: usize,
|
||||
range: HTTPRangeSpec,
|
||||
expected_copy_mode: GetObjectChunkCopyMode,
|
||||
}
|
||||
|
||||
struct EcstoreBenchEnv {
|
||||
_temp_dir: TempDir,
|
||||
store: Arc<ECStore>,
|
||||
bucket: String,
|
||||
key: String,
|
||||
range: HTTPRangeSpec,
|
||||
opts: ObjectOptions,
|
||||
expected_len: usize,
|
||||
expected_copy_mode: GetObjectChunkCopyMode,
|
||||
}
|
||||
|
||||
fn ecstore_bench_cases() -> [EcstoreBenchCase; 1] {
|
||||
[EcstoreBenchCase {
|
||||
name: "multi_disk_range",
|
||||
disk_count: 4,
|
||||
payload_len: 3 * 1024 * 1024 + 137,
|
||||
range: HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 123_457,
|
||||
end: 2 * 1024 * 1024 + 33_333,
|
||||
},
|
||||
expected_copy_mode: expected_direct_copy_mode(),
|
||||
}]
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
const fn expected_direct_copy_mode() -> GetObjectChunkCopyMode {
|
||||
GetObjectChunkCopyMode::TrueZeroCopy
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
const fn expected_direct_copy_mode() -> GetObjectChunkCopyMode {
|
||||
GetObjectChunkCopyMode::SharedBytes
|
||||
}
|
||||
|
||||
fn clone_range_spec(range: &HTTPRangeSpec) -> HTTPRangeSpec {
|
||||
HTTPRangeSpec {
|
||||
is_suffix_length: range.is_suffix_length,
|
||||
start: range.start,
|
||||
end: range.end,
|
||||
}
|
||||
}
|
||||
|
||||
fn next_loopback_addr() -> SocketAddr {
|
||||
static NEXT_PORT: AtomicU16 = AtomicU16::new(39013);
|
||||
let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed);
|
||||
SocketAddr::from(([127, 0, 0, 1], port))
|
||||
}
|
||||
|
||||
fn build_endpoint_pools(paths: &[std::path::PathBuf]) -> EndpointServerPools {
|
||||
let mut endpoints = Vec::with_capacity(paths.len());
|
||||
for (idx, disk_path) in paths.iter().enumerate() {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf8 path")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(idx);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
|
||||
EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: paths.len(),
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "bench".to_string(),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
}])
|
||||
}
|
||||
|
||||
async fn build_ecstore_bench_env(case: &EcstoreBenchCase) -> EcstoreBenchEnv {
|
||||
let temp_dir = tempfile::tempdir().expect("tempdir");
|
||||
let mut disk_paths = Vec::with_capacity(case.disk_count);
|
||||
for idx in 0..case.disk_count {
|
||||
let path = temp_dir.path().join(format!("disk{}", idx + 1));
|
||||
tokio::fs::create_dir_all(&path).await.expect("create disk dir");
|
||||
disk_paths.push(path);
|
||||
}
|
||||
|
||||
let endpoint_pools = build_endpoint_pools(&disk_paths);
|
||||
GLOBAL_LOCAL_DISK_MAP.write().await.clear();
|
||||
GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
|
||||
GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
|
||||
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
|
||||
|
||||
let store = ECStore::new(next_loopback_addr(), endpoint_pools, CancellationToken::new())
|
||||
.await
|
||||
.expect("create ecstore");
|
||||
|
||||
let buckets = store
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("list buckets")
|
||||
.into_iter()
|
||||
.map(|bucket| bucket.name)
|
||||
.collect();
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), buckets).await;
|
||||
|
||||
let object_id = case.name.replace('_', "-");
|
||||
let bucket = format!("bench-direct-{object_id}");
|
||||
let key = format!("objects/{object_id}.bin");
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("make bucket");
|
||||
|
||||
let payload: Vec<u8> = (0..case.payload_len).map(|idx| (idx % 251) as u8).collect();
|
||||
let mut reader = ChunkNativePutData::from_vec(payload);
|
||||
let put_info = store
|
||||
.put_object(&bucket, &key, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("put object");
|
||||
if case.disk_count > 1 {
|
||||
assert!(put_info.data_blocks > 1, "expected multi-data-shard object");
|
||||
}
|
||||
|
||||
let (_, expected_len) = case.range.get_offset_length(case.payload_len as i64).expect("range length");
|
||||
|
||||
EcstoreBenchEnv {
|
||||
_temp_dir: temp_dir,
|
||||
store,
|
||||
bucket,
|
||||
key,
|
||||
range: clone_range_spec(&case.range),
|
||||
opts: ObjectOptions::default(),
|
||||
expected_len: expected_len as usize,
|
||||
expected_copy_mode: case.expected_copy_mode,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_ecstore_get_object_chunks_bench(env: &EcstoreBenchEnv) -> (GetObjectChunkCopyMode, usize, usize) {
|
||||
let mut result = env
|
||||
.store
|
||||
.get_object_chunks(&env.bucket, &env.key, Some(clone_range_spec(&env.range)), HeaderMap::new(), &env.opts)
|
||||
.await
|
||||
.expect("get object chunks");
|
||||
let copy_mode = result.copy_mode;
|
||||
let mut total_len = 0usize;
|
||||
let mut chunk_count = 0usize;
|
||||
while let Some(chunk) = result.stream.next().await {
|
||||
let chunk = chunk.expect("chunk");
|
||||
total_len += chunk.len();
|
||||
chunk_count += 1;
|
||||
}
|
||||
|
||||
(copy_mode, total_len, chunk_count)
|
||||
}
|
||||
|
||||
fn build_shard_bytes(case: &BenchCase) -> Vec<Vec<Bytes>> {
|
||||
let total_len = case.block_size * case.blocks;
|
||||
let payload: Vec<u8> = (0..total_len).map(|idx| (idx % 251) as u8).collect();
|
||||
let mut shards = vec![Vec::with_capacity(case.blocks); case.data_shards];
|
||||
|
||||
for block in 0..case.blocks {
|
||||
let block_start = block * case.block_size;
|
||||
let block_slice = &payload[block_start..block_start + case.block_size];
|
||||
let shard_width = case.block_size / case.data_shards;
|
||||
for (shard_idx, shard) in shards.iter_mut().enumerate().take(case.data_shards) {
|
||||
let shard_start = shard_idx * shard_width;
|
||||
let shard_end = shard_start + shard_width;
|
||||
shard.push(Bytes::copy_from_slice(&block_slice[shard_start..shard_end]));
|
||||
}
|
||||
}
|
||||
|
||||
shards
|
||||
}
|
||||
|
||||
fn build_mapped_streams(shards: &[Vec<Bytes>]) -> Vec<BoxChunkStream> {
|
||||
shards
|
||||
.iter()
|
||||
.map(|shard_chunks| {
|
||||
let chunks: Vec<_> = shard_chunks
|
||||
.iter()
|
||||
.map(|chunk| {
|
||||
let mapped = MappedChunk::new(chunk.clone(), 0, chunk.len()).expect("mapped chunk");
|
||||
Ok::<IoChunk, std::io::Error>(IoChunk::Mapped(mapped))
|
||||
})
|
||||
.collect();
|
||||
Box::pin(stream::iter(chunks)) as BoxChunkStream
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn collect_old_assembly(
|
||||
shards: &[Vec<Bytes>],
|
||||
data_shards: usize,
|
||||
block_size: usize,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
) -> Vec<IoChunk> {
|
||||
if length == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let start_block = offset / block_size;
|
||||
let end_block = offset.saturating_add(length - 1) / block_size;
|
||||
let mut result = Vec::with_capacity(end_block - start_block + 1);
|
||||
|
||||
for block_index in start_block..=end_block {
|
||||
let block_offset = if block_index == start_block { offset % block_size } else { 0 };
|
||||
let block_length = if start_block == end_block {
|
||||
length
|
||||
} else if block_index == start_block {
|
||||
block_size - (offset % block_size)
|
||||
} else if block_index == end_block {
|
||||
(offset + length) % block_size
|
||||
} else {
|
||||
block_size
|
||||
};
|
||||
|
||||
if block_length == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let mut block = BytesMut::with_capacity(block_length);
|
||||
let mut write_left = block_length;
|
||||
let mut skip = block_offset;
|
||||
|
||||
for shard in shards.iter().take(data_shards) {
|
||||
let shard_chunk = &shard[block_index];
|
||||
if skip >= shard_chunk.len() {
|
||||
skip -= shard_chunk.len();
|
||||
continue;
|
||||
}
|
||||
|
||||
let available = &shard_chunk[skip..];
|
||||
skip = 0;
|
||||
let take = available.len().min(write_left);
|
||||
block.extend_from_slice(&available[..take]);
|
||||
write_left -= take;
|
||||
|
||||
if write_left == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
result.push(IoChunk::Shared(block.freeze()));
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
fn bench_direct_chunk_path(c: &mut Criterion) {
|
||||
let runtime = Runtime::new().expect("tokio runtime");
|
||||
let mut group = c.benchmark_group("direct_chunk_path");
|
||||
|
||||
for case in bench_cases() {
|
||||
let shard_bytes = build_shard_bytes(&case);
|
||||
group.throughput(Throughput::Bytes(case.length as u64));
|
||||
|
||||
group.bench_with_input(BenchmarkId::new("slice_forwarding", case.name), &case, |b, case| {
|
||||
b.iter(|| {
|
||||
let streams = build_mapped_streams(&shard_bytes);
|
||||
let chunks = runtime
|
||||
.block_on(collect_direct_data_shard_chunks_for_benchmark(
|
||||
streams,
|
||||
case.data_shards,
|
||||
case.block_size,
|
||||
case.blocks * case.block_size,
|
||||
false,
|
||||
case.offset,
|
||||
case.length,
|
||||
))
|
||||
.expect("collect direct chunks");
|
||||
black_box(chunks);
|
||||
});
|
||||
});
|
||||
|
||||
group.bench_with_input(BenchmarkId::new("assembled_copy", case.name), &case, |b, case| {
|
||||
b.iter(|| {
|
||||
let chunks = collect_old_assembly(&shard_bytes, case.data_shards, case.block_size, case.offset, case.length);
|
||||
black_box(chunks);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_ecstore_get_object_chunks(c: &mut Criterion) {
|
||||
let runtime = Runtime::new().expect("tokio runtime");
|
||||
let mut group = c.benchmark_group("ecstore_get_object_chunks");
|
||||
group.sample_size(10);
|
||||
|
||||
for case in ecstore_bench_cases() {
|
||||
let env = runtime.block_on(build_ecstore_bench_env(&case));
|
||||
let (copy_mode, total_len, _) = runtime.block_on(run_ecstore_get_object_chunks_bench(&env));
|
||||
assert_eq!(copy_mode, env.expected_copy_mode);
|
||||
assert_eq!(total_len, env.expected_len);
|
||||
|
||||
group.throughput(Throughput::Bytes(env.expected_len as u64));
|
||||
group.bench_with_input(BenchmarkId::new("drain", case.name), &env, |b, env| {
|
||||
b.iter(|| {
|
||||
let result = runtime.block_on(run_ecstore_get_object_chunks_bench(env));
|
||||
black_box(result);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_direct_chunk_path, bench_ecstore_get_object_chunks);
|
||||
criterion_main!(benches);
|
||||
@@ -1,393 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
|
||||
use futures_util::StreamExt;
|
||||
use http::HeaderMap;
|
||||
use rustfs_ecstore::bucket::metadata_sys;
|
||||
use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||
use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use rustfs_ecstore::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
|
||||
use rustfs_ecstore::store::{ECStore, init_local_disks};
|
||||
use rustfs_ecstore::store_api::{
|
||||
BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, GetObjectChunkCopyMode, HTTPRangeSpec, MakeBucketOptions,
|
||||
MultipartOperations, ObjectOperations, ObjectOptions,
|
||||
};
|
||||
use std::hint::black_box;
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU16, Ordering};
|
||||
use tempfile::TempDir;
|
||||
use tokio::runtime::Runtime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Clone)]
|
||||
enum ReconstructedReadSpec {
|
||||
PartNumber {
|
||||
part_number: usize,
|
||||
missing_part_name: &'static str,
|
||||
},
|
||||
Range {
|
||||
start: u64,
|
||||
end: u64,
|
||||
missing_part_name: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ReconstructedBenchCase {
|
||||
object_id: &'static str,
|
||||
name: &'static str,
|
||||
read_spec: ReconstructedReadSpec,
|
||||
}
|
||||
|
||||
struct ReconstructedBenchEnv {
|
||||
store: Arc<ECStore>,
|
||||
bucket: String,
|
||||
key: String,
|
||||
range: HTTPRangeSpec,
|
||||
opts: ObjectOptions,
|
||||
expected_len: usize,
|
||||
}
|
||||
|
||||
struct ReconstructedBenchSuite {
|
||||
_temp_dir: TempDir,
|
||||
store: Arc<ECStore>,
|
||||
disk_paths: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
const MULTIPART_PART_ONE_LEN: usize = 5 * 1024 * 1024;
|
||||
const MULTIPART_PART_TWO_LEN: usize = 5 * 1024 * 1024 + 137;
|
||||
const MULTIPART_PART_THREE_LEN: usize = 1024 * 1024 + 77;
|
||||
|
||||
fn reconstructed_bench_cases() -> [ReconstructedBenchCase; 4] {
|
||||
[
|
||||
ReconstructedBenchCase {
|
||||
object_id: "mp-part2",
|
||||
name: "multi_disk_missing_shard_multipart_part2",
|
||||
read_spec: ReconstructedReadSpec::PartNumber {
|
||||
part_number: 2,
|
||||
missing_part_name: "part.2",
|
||||
},
|
||||
},
|
||||
ReconstructedBenchCase {
|
||||
object_id: "mp-cross-range",
|
||||
name: "multi_disk_missing_shard_multipart_cross_part_range",
|
||||
read_spec: ReconstructedReadSpec::Range {
|
||||
start: (MULTIPART_PART_ONE_LEN - 32 * 1024) as u64,
|
||||
end: (MULTIPART_PART_ONE_LEN + 96 * 1024) as u64,
|
||||
missing_part_name: "part.2",
|
||||
},
|
||||
},
|
||||
ReconstructedBenchCase {
|
||||
object_id: "mp-part3",
|
||||
name: "multi_disk_missing_shard_multipart_part3",
|
||||
read_spec: ReconstructedReadSpec::PartNumber {
|
||||
part_number: 3,
|
||||
missing_part_name: "part.3",
|
||||
},
|
||||
},
|
||||
ReconstructedBenchCase {
|
||||
object_id: "mp-cross-final-range",
|
||||
name: "multi_disk_missing_shard_multipart_cross_final_part_range",
|
||||
read_spec: ReconstructedReadSpec::Range {
|
||||
start: (MULTIPART_PART_ONE_LEN + MULTIPART_PART_TWO_LEN - 32 * 1024) as u64,
|
||||
end: (MULTIPART_PART_ONE_LEN + MULTIPART_PART_TWO_LEN + 96 * 1024) as u64,
|
||||
missing_part_name: "part.3",
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn next_loopback_addr() -> SocketAddr {
|
||||
static NEXT_PORT: AtomicU16 = AtomicU16::new(39113);
|
||||
let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed);
|
||||
SocketAddr::from(([127, 0, 0, 1], port))
|
||||
}
|
||||
|
||||
fn clone_range_spec(range: &HTTPRangeSpec) -> HTTPRangeSpec {
|
||||
HTTPRangeSpec {
|
||||
is_suffix_length: range.is_suffix_length,
|
||||
start: range.start,
|
||||
end: range.end,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_endpoint_pools(paths: &[PathBuf]) -> EndpointServerPools {
|
||||
let mut endpoints = Vec::with_capacity(paths.len());
|
||||
for (idx, disk_path) in paths.iter().enumerate() {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf8 path")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(idx);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
|
||||
EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: paths.len(),
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "bench".to_string(),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
}])
|
||||
}
|
||||
|
||||
fn find_part_files(root: &Path, part_name: &str, out: &mut Vec<PathBuf>) {
|
||||
let Ok(entries) = std::fs::read_dir(root) else {
|
||||
return;
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
find_part_files(&path, part_name, out);
|
||||
continue;
|
||||
}
|
||||
|
||||
if path.file_name().and_then(|name| name.to_str()) == Some(part_name) {
|
||||
out.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_part_files(root: &Path, part_name: &str) -> Vec<(PathBuf, Vec<u8>)> {
|
||||
let mut paths = Vec::new();
|
||||
find_part_files(root, part_name, &mut paths);
|
||||
|
||||
let mut removed = Vec::with_capacity(paths.len());
|
||||
for path in paths {
|
||||
let content = tokio::fs::read(&path).await.expect("read part file before removal");
|
||||
tokio::fs::remove_file(&path).await.expect("remove part file");
|
||||
removed.push((path, content));
|
||||
}
|
||||
|
||||
removed
|
||||
}
|
||||
|
||||
async fn restore_part_files(files: Vec<(PathBuf, Vec<u8>)>) {
|
||||
for (path, content) in files {
|
||||
if let Some(parent) = path.parent() {
|
||||
tokio::fs::create_dir_all(parent)
|
||||
.await
|
||||
.expect("ensure parent for part restore");
|
||||
}
|
||||
tokio::fs::write(&path, content).await.expect("restore part file");
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_reconstructed_get_object_chunks(
|
||||
store: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
range: &HTTPRangeSpec,
|
||||
opts: &ObjectOptions,
|
||||
) -> (GetObjectChunkCopyMode, usize, usize) {
|
||||
let mut result = store
|
||||
.get_object_chunks(bucket, key, Some(clone_range_spec(range)), HeaderMap::new(), opts)
|
||||
.await
|
||||
.expect("get object chunks");
|
||||
|
||||
let copy_mode = result.copy_mode;
|
||||
let mut total_len = 0usize;
|
||||
let mut chunk_count = 0usize;
|
||||
while let Some(chunk) = result.stream.next().await {
|
||||
let chunk = chunk.expect("chunk");
|
||||
total_len += chunk.len();
|
||||
chunk_count += 1;
|
||||
}
|
||||
|
||||
(copy_mode, total_len, chunk_count)
|
||||
}
|
||||
|
||||
async fn create_multipart_object(store: &Arc<ECStore>, bucket: &str, key: &str, parts: &[Vec<u8>]) -> usize {
|
||||
let upload = store
|
||||
.new_multipart_upload(bucket, key, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("new multipart upload");
|
||||
|
||||
let mut completed_parts = Vec::with_capacity(parts.len());
|
||||
for (idx, part) in parts.iter().enumerate() {
|
||||
let mut reader = ChunkNativePutData::from_vec(part.clone());
|
||||
let part_info = store
|
||||
.put_object_part(bucket, key, &upload.upload_id, idx + 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("put object part");
|
||||
completed_parts.push(CompletePart {
|
||||
part_num: idx + 1,
|
||||
etag: part_info.etag,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, key, &upload.upload_id, completed_parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("complete multipart upload");
|
||||
|
||||
parts.iter().map(Vec::len).sum()
|
||||
}
|
||||
|
||||
async fn build_reconstructed_bench_suite() -> ReconstructedBenchSuite {
|
||||
let temp_dir = tempfile::tempdir().expect("tempdir");
|
||||
let disk_paths: Vec<_> = (1..=4).map(|idx| temp_dir.path().join(format!("disk{idx}"))).collect();
|
||||
for disk_path in &disk_paths {
|
||||
tokio::fs::create_dir_all(disk_path).await.expect("create disk dir");
|
||||
}
|
||||
|
||||
let endpoint_pools = build_endpoint_pools(&disk_paths);
|
||||
GLOBAL_LOCAL_DISK_MAP.write().await.clear();
|
||||
GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
|
||||
GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
|
||||
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
|
||||
|
||||
let store = ECStore::new(next_loopback_addr(), endpoint_pools, CancellationToken::new())
|
||||
.await
|
||||
.expect("create ecstore");
|
||||
|
||||
let buckets = store
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("list buckets")
|
||||
.into_iter()
|
||||
.map(|bucket| bucket.name)
|
||||
.collect();
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), buckets).await;
|
||||
|
||||
ReconstructedBenchSuite {
|
||||
_temp_dir: temp_dir,
|
||||
store,
|
||||
disk_paths,
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_reconstructed_bench_env(suite: &ReconstructedBenchSuite, case: &ReconstructedBenchCase) -> ReconstructedBenchEnv {
|
||||
let bucket = format!("bench-r-{}", case.object_id);
|
||||
let key = format!("objects/{}.bin", case.object_id);
|
||||
suite
|
||||
.store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("make bucket");
|
||||
|
||||
let part_one: Vec<u8> = (0..MULTIPART_PART_ONE_LEN).map(|idx| (idx % 251) as u8).collect();
|
||||
let part_two: Vec<u8> = (0..MULTIPART_PART_TWO_LEN).map(|idx| ((idx + 11) % 251) as u8).collect();
|
||||
let part_three: Vec<u8> = (0..MULTIPART_PART_THREE_LEN).map(|idx| ((idx + 29) % 251) as u8).collect();
|
||||
let payload_len = create_multipart_object(&suite.store, &bucket, &key, &[part_one, part_two, part_three]).await;
|
||||
let info = suite
|
||||
.store
|
||||
.get_object_info(&bucket, &key, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("get object info");
|
||||
let (range, opts, missing_part_name) = match case.read_spec.clone() {
|
||||
ReconstructedReadSpec::PartNumber {
|
||||
part_number,
|
||||
missing_part_name,
|
||||
} => {
|
||||
let range = HTTPRangeSpec::from_object_info(&info, part_number).expect("part_number range");
|
||||
let opts = ObjectOptions {
|
||||
part_number: Some(part_number),
|
||||
..Default::default()
|
||||
};
|
||||
(range, opts, missing_part_name)
|
||||
}
|
||||
ReconstructedReadSpec::Range {
|
||||
start,
|
||||
end,
|
||||
missing_part_name,
|
||||
} => (
|
||||
HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: start as i64,
|
||||
end: end as i64,
|
||||
},
|
||||
ObjectOptions::default(),
|
||||
missing_part_name,
|
||||
),
|
||||
};
|
||||
|
||||
let (_, expected_len) = range.get_offset_length(payload_len as i64).expect("range length");
|
||||
let mut selected_reconstructed = false;
|
||||
for disk_path in &suite.disk_paths {
|
||||
let object_root = disk_path
|
||||
.join(&bucket)
|
||||
.join("objects")
|
||||
.join(format!("{}.bin", case.object_id));
|
||||
let removed_parts = remove_part_files(&object_root, missing_part_name).await;
|
||||
if removed_parts.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let (copy_mode, total_len, _) = run_reconstructed_get_object_chunks(&suite.store, &bucket, &key, &range, &opts).await;
|
||||
if copy_mode == GetObjectChunkCopyMode::Reconstructed && total_len == expected_len as usize {
|
||||
selected_reconstructed = true;
|
||||
break;
|
||||
}
|
||||
|
||||
restore_part_files(removed_parts).await;
|
||||
}
|
||||
assert!(
|
||||
selected_reconstructed,
|
||||
"failed to select a missing shard placement that triggers reconstructed copy mode for benchmark case {}",
|
||||
case.name
|
||||
);
|
||||
|
||||
ReconstructedBenchEnv {
|
||||
store: suite.store.clone(),
|
||||
bucket,
|
||||
key,
|
||||
range: clone_range_spec(&range),
|
||||
opts,
|
||||
expected_len: expected_len as usize,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_reconstructed_get_object_chunks_bench(env: &ReconstructedBenchEnv) -> (GetObjectChunkCopyMode, usize, usize) {
|
||||
run_reconstructed_get_object_chunks(&env.store, &env.bucket, &env.key, &env.range, &env.opts).await
|
||||
}
|
||||
|
||||
fn bench_reconstructed_chunk_path(c: &mut Criterion) {
|
||||
let runtime = Runtime::new().expect("tokio runtime");
|
||||
let suite = runtime.block_on(build_reconstructed_bench_suite());
|
||||
let mut group = c.benchmark_group("reconstructed_chunk_path");
|
||||
group.sample_size(10);
|
||||
for case in reconstructed_bench_cases() {
|
||||
let env = runtime.block_on(build_reconstructed_bench_env(&suite, &case));
|
||||
let (copy_mode, total_len, _) = runtime.block_on(run_reconstructed_get_object_chunks_bench(&env));
|
||||
assert_eq!(copy_mode, GetObjectChunkCopyMode::Reconstructed);
|
||||
assert_eq!(total_len, env.expected_len);
|
||||
|
||||
group.throughput(Throughput::Bytes(env.expected_len as u64));
|
||||
group.bench_with_input(BenchmarkId::new("drain", case.name), &env, |b, env| {
|
||||
b.iter(|| {
|
||||
let result = runtime.block_on(run_reconstructed_get_object_chunks_bench(env));
|
||||
black_box(result);
|
||||
});
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_reconstructed_chunk_path);
|
||||
criterion_main!(benches);
|
||||
@@ -119,16 +119,6 @@ run_large_data_test() {
|
||||
print_success "Large-dataset tests completed"
|
||||
}
|
||||
|
||||
# Run direct chunk path benchmarks
|
||||
run_direct_chunk_benchmark() {
|
||||
print_info "📦 Starting direct chunk path benchmarks..."
|
||||
echo "================================================"
|
||||
|
||||
cargo bench --bench direct_chunk_benchmark
|
||||
|
||||
print_success "Direct chunk path benchmarks completed"
|
||||
}
|
||||
|
||||
# Generate comparison report
|
||||
generate_comparison_report() {
|
||||
print_info "📊 Generating performance report..."
|
||||
@@ -178,7 +168,6 @@ show_help() {
|
||||
echo " full Run the full benchmark suite"
|
||||
echo " performance Run detailed performance tests"
|
||||
echo " simd Run the SIMD-only tests"
|
||||
echo " direct Run the direct chunk path benchmarks"
|
||||
echo " large Run large-dataset tests"
|
||||
echo " clean Remove previous results"
|
||||
echo " help Show this help message"
|
||||
@@ -188,7 +177,6 @@ show_help() {
|
||||
echo " $0 performance # Detailed performance test"
|
||||
echo " $0 full # Full benchmark suite"
|
||||
echo " $0 simd # SIMD-only benchmark"
|
||||
echo " $0 direct # Direct chunk path benchmark"
|
||||
echo " $0 large # Large-dataset benchmark"
|
||||
echo ""
|
||||
echo "Features:"
|
||||
@@ -252,11 +240,6 @@ main() {
|
||||
run_simd_benchmark
|
||||
generate_comparison_report
|
||||
;;
|
||||
"direct")
|
||||
cleanup
|
||||
run_direct_chunk_benchmark
|
||||
generate_comparison_report
|
||||
;;
|
||||
"large")
|
||||
cleanup
|
||||
run_large_data_test
|
||||
|
||||
@@ -162,8 +162,9 @@ pub async fn get_local_server_property() -> ServerProperties {
|
||||
|
||||
let mut props = ServerProperties {
|
||||
endpoint: addr,
|
||||
uptime: SystemTime::now()
|
||||
.duration_since(*GLOBAL_BOOT_TIME.get().unwrap())
|
||||
uptime: GLOBAL_BOOT_TIME
|
||||
.get()
|
||||
.and_then(|boot_time| SystemTime::now().duration_since(*boot_time).ok())
|
||||
.unwrap_or_default()
|
||||
.as_secs(),
|
||||
network,
|
||||
|
||||
+11
-781
@@ -14,328 +14,13 @@
|
||||
|
||||
use crate::disk::{self, DiskAPI as _, DiskStore, error::DiskError};
|
||||
use crate::erasure_coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
|
||||
use crate::store_api::{GetObjectChunkCopyMode, GetObjectChunkPath, GetObjectChunkResult};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures_util::{StreamExt, stream};
|
||||
use rustfs_io_core::{BoxChunkStream, IoChunk};
|
||||
use bytes::Bytes;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use std::collections::VecDeque;
|
||||
use std::io::Cursor;
|
||||
use std::time::Instant;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::debug;
|
||||
|
||||
const BITROT_READ_OPERATION: &str = "bitrot_read";
|
||||
|
||||
fn classify_chunk_copy_mode(source_direct: bool, copied: bool) -> GetObjectChunkCopyMode {
|
||||
if copied {
|
||||
GetObjectChunkCopyMode::SingleCopy
|
||||
} else if source_direct {
|
||||
GetObjectChunkCopyMode::TrueZeroCopy
|
||||
} else {
|
||||
GetObjectChunkCopyMode::SharedBytes
|
||||
}
|
||||
}
|
||||
|
||||
struct ChunkSpan {
|
||||
bytes: Bytes,
|
||||
chunk: IoChunk,
|
||||
copied: bool,
|
||||
}
|
||||
|
||||
fn take_contiguous_chunk_span(chunk: &IoChunk, offset: usize, len: usize) -> std::io::Result<ChunkSpan> {
|
||||
match chunk {
|
||||
IoChunk::Shared(bytes) => {
|
||||
let bytes = bytes.slice(offset..offset + len);
|
||||
Ok(ChunkSpan {
|
||||
bytes: bytes.clone(),
|
||||
chunk: IoChunk::Shared(bytes),
|
||||
copied: false,
|
||||
})
|
||||
}
|
||||
IoChunk::Mapped(mapped) => {
|
||||
let chunk = IoChunk::Mapped(mapped.slice(offset, len)?);
|
||||
let bytes = chunk.as_bytes();
|
||||
Ok(ChunkSpan {
|
||||
bytes,
|
||||
chunk,
|
||||
copied: false,
|
||||
})
|
||||
}
|
||||
IoChunk::Pooled(pooled) => {
|
||||
let chunk = IoChunk::Pooled(pooled.slice(offset, len)?);
|
||||
let bytes = chunk.as_bytes();
|
||||
Ok(ChunkSpan {
|
||||
bytes,
|
||||
chunk,
|
||||
copied: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BitrotChunkSource {
|
||||
source_stream: BoxChunkStream,
|
||||
source_chunks: VecDeque<IoChunk>,
|
||||
source_chunk_offset: usize,
|
||||
source_buffered_bytes: usize,
|
||||
source_done: bool,
|
||||
}
|
||||
|
||||
struct BitrotChunkStreamState {
|
||||
source: BitrotChunkSource,
|
||||
decoded_remaining: usize,
|
||||
trim_prefix: usize,
|
||||
output_remaining: usize,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
skip_verify: bool,
|
||||
}
|
||||
|
||||
struct ChunkCursor<'a> {
|
||||
chunks: &'a [IoChunk],
|
||||
chunk_index: usize,
|
||||
chunk_offset: usize,
|
||||
consumed: usize,
|
||||
total_len: usize,
|
||||
}
|
||||
|
||||
impl<'a> ChunkCursor<'a> {
|
||||
fn new(chunks: &'a [IoChunk]) -> Self {
|
||||
Self {
|
||||
chunks,
|
||||
chunk_index: 0,
|
||||
chunk_offset: 0,
|
||||
consumed: 0,
|
||||
total_len: chunks.iter().map(IoChunk::len).sum(),
|
||||
}
|
||||
}
|
||||
|
||||
fn remaining(&self) -> usize {
|
||||
self.total_len.saturating_sub(self.consumed)
|
||||
}
|
||||
|
||||
fn skip_empty_chunks(&mut self) {
|
||||
while let Some(chunk) = self.chunks.get(self.chunk_index) {
|
||||
if self.chunk_offset < chunk.len() {
|
||||
break;
|
||||
}
|
||||
self.chunk_index += 1;
|
||||
self.chunk_offset = 0;
|
||||
}
|
||||
}
|
||||
|
||||
fn advance(&mut self, len: usize) {
|
||||
self.consumed += len;
|
||||
self.chunk_offset += len;
|
||||
self.skip_empty_chunks();
|
||||
}
|
||||
|
||||
fn take_span(&mut self, len: usize) -> std::io::Result<ChunkSpan> {
|
||||
self.skip_empty_chunks();
|
||||
if self.remaining() < len {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
|
||||
}
|
||||
|
||||
let Some(chunk) = self.chunks.get(self.chunk_index) else {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "missing bitrot chunk source"));
|
||||
};
|
||||
let available = chunk.len().saturating_sub(self.chunk_offset);
|
||||
|
||||
if len <= available {
|
||||
let span = take_contiguous_chunk_span(chunk, self.chunk_offset, len)?;
|
||||
self.advance(len);
|
||||
return Ok(span);
|
||||
}
|
||||
|
||||
let mut aggregate = BytesMut::with_capacity(len);
|
||||
let mut remaining = len;
|
||||
while remaining > 0 {
|
||||
self.skip_empty_chunks();
|
||||
let Some(chunk) = self.chunks.get(self.chunk_index) else {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
|
||||
};
|
||||
let available = chunk.len().saturating_sub(self.chunk_offset);
|
||||
let take = available.min(remaining);
|
||||
aggregate.extend_from_slice(&chunk.as_bytes()[self.chunk_offset..self.chunk_offset + take]);
|
||||
self.advance(take);
|
||||
remaining -= take;
|
||||
}
|
||||
|
||||
let bytes = aggregate.freeze();
|
||||
Ok(ChunkSpan {
|
||||
bytes: bytes.clone(),
|
||||
chunk: IoChunk::Shared(bytes),
|
||||
copied: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl BitrotChunkSource {
|
||||
fn new(source_stream: BoxChunkStream, source_chunks: VecDeque<IoChunk>, source_done: bool) -> Self {
|
||||
let source_buffered_bytes = source_chunks.iter().map(IoChunk::len).sum();
|
||||
Self {
|
||||
source_stream,
|
||||
source_chunks,
|
||||
source_chunk_offset: 0,
|
||||
source_buffered_bytes,
|
||||
source_done,
|
||||
}
|
||||
}
|
||||
|
||||
fn skip_empty_chunks(&mut self) {
|
||||
while let Some(chunk) = self.source_chunks.front() {
|
||||
if self.source_chunk_offset < chunk.len() {
|
||||
break;
|
||||
}
|
||||
self.source_chunks.pop_front();
|
||||
self.source_chunk_offset = 0;
|
||||
}
|
||||
}
|
||||
|
||||
async fn fill(&mut self, min_bytes: usize) -> std::io::Result<()> {
|
||||
while self.source_buffered_bytes < min_bytes && !self.source_done {
|
||||
match self.source_stream.next().await {
|
||||
Some(Ok(chunk)) => {
|
||||
self.source_buffered_bytes += chunk.len();
|
||||
self.source_chunks.push_back(chunk);
|
||||
}
|
||||
Some(Err(err)) => return Err(err),
|
||||
None => self.source_done = true,
|
||||
}
|
||||
}
|
||||
|
||||
if self.source_buffered_bytes < min_bytes {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn advance(&mut self, len: usize) {
|
||||
self.source_buffered_bytes = self.source_buffered_bytes.saturating_sub(len);
|
||||
self.source_chunk_offset += len;
|
||||
self.skip_empty_chunks();
|
||||
}
|
||||
|
||||
fn take_span(&mut self, len: usize) -> std::io::Result<ChunkSpan> {
|
||||
self.skip_empty_chunks();
|
||||
if self.source_buffered_bytes < len {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
|
||||
}
|
||||
|
||||
let Some(chunk) = self.source_chunks.front() else {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "missing bitrot chunk source"));
|
||||
};
|
||||
let available = chunk.len().saturating_sub(self.source_chunk_offset);
|
||||
|
||||
if len <= available {
|
||||
let span = take_contiguous_chunk_span(chunk, self.source_chunk_offset, len)?;
|
||||
self.advance(len);
|
||||
return Ok(span);
|
||||
}
|
||||
|
||||
let mut aggregate = BytesMut::with_capacity(len);
|
||||
let mut remaining = len;
|
||||
while remaining > 0 {
|
||||
self.skip_empty_chunks();
|
||||
let Some(chunk) = self.source_chunks.front() else {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
|
||||
};
|
||||
let available = chunk.len().saturating_sub(self.source_chunk_offset);
|
||||
let take = available.min(remaining);
|
||||
aggregate.extend_from_slice(&chunk.as_bytes()[self.source_chunk_offset..self.source_chunk_offset + take]);
|
||||
self.advance(take);
|
||||
remaining -= take;
|
||||
}
|
||||
|
||||
let bytes = aggregate.freeze();
|
||||
Ok(ChunkSpan {
|
||||
bytes: bytes.clone(),
|
||||
chunk: IoChunk::Shared(bytes),
|
||||
copied: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl BitrotChunkStreamState {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn new(
|
||||
source_stream: BoxChunkStream,
|
||||
source_chunks: VecDeque<IoChunk>,
|
||||
source_done: bool,
|
||||
decoded_remaining: usize,
|
||||
trim_prefix: usize,
|
||||
output_remaining: usize,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
skip_verify: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
source: BitrotChunkSource::new(source_stream, source_chunks, source_done),
|
||||
decoded_remaining,
|
||||
trim_prefix,
|
||||
output_remaining,
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
skip_verify,
|
||||
}
|
||||
}
|
||||
|
||||
fn hash_size(&self) -> usize {
|
||||
self.checksum_algo.size()
|
||||
}
|
||||
|
||||
async fn next_verified_chunk(&mut self) -> std::io::Result<Option<IoChunk>> {
|
||||
let hash_size = self.hash_size();
|
||||
|
||||
while self.output_remaining > 0 && self.decoded_remaining > 0 {
|
||||
let data_len = self.shard_size.min(self.decoded_remaining);
|
||||
|
||||
let expected_hash = if hash_size > 0 {
|
||||
self.source.fill(hash_size).await?;
|
||||
Some(self.source.take_span(hash_size)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
self.source.fill(data_len).await?;
|
||||
let data_span = self.source.take_span(data_len)?;
|
||||
|
||||
if let Some(expected_hash) = expected_hash
|
||||
&& !self.skip_verify
|
||||
&& self.checksum_algo.hash_encode(data_span.bytes.as_ref()).as_ref() != expected_hash.bytes.as_ref()
|
||||
{
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
|
||||
}
|
||||
|
||||
self.decoded_remaining -= data_len;
|
||||
|
||||
if self.trim_prefix >= data_len {
|
||||
self.trim_prefix -= data_len;
|
||||
continue;
|
||||
}
|
||||
|
||||
let start = self.trim_prefix;
|
||||
self.trim_prefix = 0;
|
||||
let take = (data_len - start).min(self.output_remaining);
|
||||
self.output_remaining -= take;
|
||||
|
||||
let chunk = if start == 0 && take == data_len {
|
||||
data_span.chunk
|
||||
} else {
|
||||
data_span.chunk.slice(start, take)?
|
||||
};
|
||||
|
||||
if !chunk.is_empty() {
|
||||
return Ok(Some(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a BitrotReader from either inline data or disk file stream
|
||||
///
|
||||
/// # Parameters
|
||||
@@ -379,40 +64,21 @@ pub async fn create_bitrot_reader(
|
||||
Ok(Some(reader))
|
||||
} else if let Some(disk) = disk {
|
||||
// Read from disk
|
||||
if use_zero_copy {
|
||||
if !disk.is_local() {
|
||||
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
|
||||
rustfs_io_metrics::record_io_fallback(
|
||||
rustfs_io_metrics::IoStage::ReadSetup,
|
||||
rustfs_io_metrics::FallbackReason::NonLocalBackend,
|
||||
);
|
||||
|
||||
let rd = disk.read_file_stream(bucket, path, offset, length).await?;
|
||||
let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
|
||||
return Ok(Some(reader));
|
||||
}
|
||||
|
||||
if use_zero_copy && disk.is_local() {
|
||||
// Try zero-copy read first (uses mmap on Unix)
|
||||
let start = Instant::now();
|
||||
match disk.read_file_zero_copy(bucket, path, offset, length).await {
|
||||
Ok(bytes) => {
|
||||
let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Fast);
|
||||
// `read_file_zero_copy()` returns a shared `Bytes` view, but it may still
|
||||
// internally aggregate multiple chunk windows. The exact chunk-native copy
|
||||
// mode is only preserved by `create_bitrot_chunk_stream()`.
|
||||
rustfs_io_metrics::record_io_copy_mode(
|
||||
BITROT_READ_OPERATION,
|
||||
rustfs_io_metrics::CopyMode::SharedBytes,
|
||||
bytes.len(),
|
||||
);
|
||||
// Record zero-copy metrics
|
||||
rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms);
|
||||
|
||||
// Log successful zero-copy read
|
||||
debug!(
|
||||
size = bytes.len(),
|
||||
duration_ms,
|
||||
path = %path,
|
||||
"bitrot_fast_read_success"
|
||||
"zero_copy_read_success"
|
||||
);
|
||||
|
||||
// Wrap Bytes in Cursor for AsyncRead
|
||||
@@ -427,16 +93,14 @@ pub async fn create_bitrot_reader(
|
||||
Ok(Some(reader))
|
||||
}
|
||||
Err(e) => {
|
||||
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
|
||||
rustfs_io_metrics::record_io_fallback(
|
||||
rustfs_io_metrics::IoStage::ReadSetup,
|
||||
rustfs_io_metrics::FallbackReason::Unknown,
|
||||
);
|
||||
// Record zero-copy fallback
|
||||
rustfs_io_metrics::record_zero_copy_fallback(&format!("{:?}", e));
|
||||
|
||||
// Log zero-copy fallback
|
||||
debug!(
|
||||
reason = %e,
|
||||
reason = %format!("{:?}", e),
|
||||
path = %path,
|
||||
"bitrot_fast_read_fallback"
|
||||
"zero_copy_fallback"
|
||||
);
|
||||
|
||||
// Fall back to regular stream read on error
|
||||
@@ -453,7 +117,6 @@ pub async fn create_bitrot_reader(
|
||||
}
|
||||
}
|
||||
} else {
|
||||
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
|
||||
// Use regular stream read
|
||||
match disk.read_file_stream(bucket, path, offset, length).await {
|
||||
Ok(rd) => {
|
||||
@@ -469,198 +132,6 @@ pub async fn create_bitrot_reader(
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a chunk stream from bitrot-encoded data, preserving source chunk provenance when possible.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn create_bitrot_chunk_stream(
|
||||
inline_data: Option<&[u8]>,
|
||||
disk: Option<&DiskStore>,
|
||||
bucket: &str,
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
total_data_size: usize,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
skip_verify: bool,
|
||||
use_zero_copy: bool,
|
||||
) -> disk::error::Result<Option<GetObjectChunkResult>> {
|
||||
let fetch_start = (offset / shard_size) * shard_size;
|
||||
let fetch_end = (offset + length).div_ceil(shard_size) * shard_size;
|
||||
let fetch_end = fetch_end.min(total_data_size);
|
||||
let fetch_length = fetch_end.saturating_sub(fetch_start);
|
||||
let trim_prefix = offset.saturating_sub(fetch_start);
|
||||
let hash_size = checksum_algo.size();
|
||||
let encoded_length = fetch_length.div_ceil(shard_size) * hash_size + fetch_length;
|
||||
let encoded_offset = fetch_start.div_ceil(shard_size) * hash_size + fetch_start;
|
||||
|
||||
let mut source_done = false;
|
||||
let (source_stream, mut prefetched_chunks, source_direct) = if let Some(data) = inline_data {
|
||||
source_done = true;
|
||||
let mut chunks = VecDeque::new();
|
||||
chunks.push_back(IoChunk::Shared(
|
||||
Bytes::copy_from_slice(data).slice(encoded_offset..encoded_offset + encoded_length),
|
||||
));
|
||||
let source_stream: BoxChunkStream = Box::pin(stream::empty::<std::io::Result<IoChunk>>());
|
||||
(source_stream, chunks, false)
|
||||
} else if let Some(disk) = disk {
|
||||
if use_zero_copy {
|
||||
let mut source_stream = disk.read_file_chunks(bucket, path, encoded_offset, encoded_length).await?;
|
||||
let mut prefetched_chunks = VecDeque::new();
|
||||
let mut direct = true;
|
||||
while prefetched_chunks.len() < 2 {
|
||||
let Some(chunk) = source_stream.next().await else {
|
||||
source_done = true;
|
||||
break;
|
||||
};
|
||||
let chunk = chunk?;
|
||||
direct &= matches!(chunk, IoChunk::Mapped(_));
|
||||
prefetched_chunks.push_back(chunk);
|
||||
}
|
||||
(source_stream, prefetched_chunks, direct)
|
||||
} else {
|
||||
source_done = true;
|
||||
let bytes = disk.read_file_zero_copy(bucket, path, encoded_offset, encoded_length).await?;
|
||||
let mut chunks = VecDeque::new();
|
||||
chunks.push_back(IoChunk::Shared(bytes));
|
||||
let source_stream: BoxChunkStream = Box::pin(stream::empty::<std::io::Result<IoChunk>>());
|
||||
(source_stream, chunks, false)
|
||||
}
|
||||
} else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let copied = predicted_stream_copy(encoded_length, shard_size, checksum_algo.size(), &prefetched_chunks, source_done);
|
||||
let state = BitrotChunkStreamState::new(
|
||||
source_stream,
|
||||
std::mem::take(&mut prefetched_chunks),
|
||||
source_done,
|
||||
fetch_length,
|
||||
trim_prefix,
|
||||
length,
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
skip_verify,
|
||||
);
|
||||
let stream = stream::unfold(Some(state), |state| async move {
|
||||
let mut state = match state {
|
||||
Some(state) => state,
|
||||
None => return None,
|
||||
};
|
||||
|
||||
match state.next_verified_chunk().await {
|
||||
Ok(Some(chunk)) => {
|
||||
let next_state = if state.output_remaining == 0 { None } else { Some(state) };
|
||||
Some((Ok::<IoChunk, std::io::Error>(chunk), next_state))
|
||||
}
|
||||
Ok(None) => None,
|
||||
Err(err) => Some((Err(err), None)),
|
||||
}
|
||||
});
|
||||
Ok(Some(GetObjectChunkResult {
|
||||
stream: Box::pin(stream),
|
||||
path: GetObjectChunkPath::Direct,
|
||||
copy_mode: classify_chunk_copy_mode(source_direct, copied),
|
||||
}))
|
||||
}
|
||||
|
||||
fn predicted_stream_copy(
|
||||
encoded_length: usize,
|
||||
shard_size: usize,
|
||||
hash_size: usize,
|
||||
prefetched_chunks: &VecDeque<IoChunk>,
|
||||
source_done: bool,
|
||||
) -> bool {
|
||||
if prefetched_chunks.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
if source_done && prefetched_chunks.len() == 1 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let full_frame_len = hash_size + shard_size;
|
||||
if full_frame_len == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let first_window_len = prefetched_chunks.front().map(IoChunk::len).unwrap_or(encoded_length);
|
||||
encoded_length > first_window_len && !first_window_len.is_multiple_of(full_frame_len)
|
||||
}
|
||||
|
||||
fn trim_chunk_vec(chunks: Vec<IoChunk>, offset: usize, length: usize) -> std::io::Result<Vec<IoChunk>> {
|
||||
let mut skip = offset;
|
||||
let mut remaining = length;
|
||||
let mut result = Vec::new();
|
||||
|
||||
for chunk in chunks {
|
||||
if remaining == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let chunk_len = chunk.len();
|
||||
if skip >= chunk_len {
|
||||
skip -= chunk_len;
|
||||
continue;
|
||||
}
|
||||
|
||||
let start = skip;
|
||||
let take = (chunk_len - start).min(remaining);
|
||||
result.push(chunk.slice(start, take)?);
|
||||
remaining -= take;
|
||||
skip = 0;
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn decode_bitrot_chunk_source(
|
||||
source_chunks: &[IoChunk],
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
skip_verify: bool,
|
||||
) -> std::io::Result<(Vec<IoChunk>, bool)> {
|
||||
let hash_size = checksum_algo.size();
|
||||
let mut cursor = ChunkCursor::new(source_chunks);
|
||||
let mut result = Vec::new();
|
||||
let mut copied = false;
|
||||
|
||||
while cursor.remaining() > 0 {
|
||||
let expected_hash = if hash_size > 0 {
|
||||
Some(cursor.take_span(hash_size)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let data_len = shard_size.min(cursor.remaining());
|
||||
if data_len == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let data_span = cursor.take_span(data_len)?;
|
||||
copied |= data_span.copied;
|
||||
if let Some(expected_hash) = expected_hash {
|
||||
copied |= expected_hash.copied;
|
||||
if !skip_verify && checksum_algo.hash_encode(data_span.bytes.as_ref()).as_ref() != expected_hash.bytes.as_ref() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
|
||||
}
|
||||
}
|
||||
|
||||
result.push(data_span.chunk);
|
||||
}
|
||||
|
||||
Ok((result, copied))
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn decode_bitrot_chunk_source_for_bench(
|
||||
source_chunks: &[IoChunk],
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
skip_verify: bool,
|
||||
) -> std::io::Result<(Vec<IoChunk>, bool)> {
|
||||
decode_bitrot_chunk_source(source_chunks, shard_size, checksum_algo, skip_verify)
|
||||
}
|
||||
|
||||
/// Create a new BitrotWriterWrapper based on the provided parameters
|
||||
///
|
||||
/// # Parameters
|
||||
@@ -705,7 +176,6 @@ pub async fn create_bitrot_writer(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use futures_util::StreamExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_reader_with_inline_data() {
|
||||
@@ -756,246 +226,6 @@ mod tests {
|
||||
assert!(result.unwrap().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_chunk_stream_with_inline_data() {
|
||||
let shard_size = 4;
|
||||
let checksum_algo = HashAlgorithm::HighwayHash256S;
|
||||
let shard1 = b"abcd";
|
||||
let shard2 = b"ef";
|
||||
|
||||
let mut encoded = Vec::new();
|
||||
encoded.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
|
||||
encoded.extend_from_slice(shard1);
|
||||
encoded.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
|
||||
encoded.extend_from_slice(shard2);
|
||||
|
||||
let mut stream = create_bitrot_chunk_stream(
|
||||
Some(&encoded),
|
||||
None,
|
||||
"test-bucket",
|
||||
"test-path",
|
||||
0,
|
||||
shard1.len() + shard2.len(),
|
||||
shard1.len() + shard2.len(),
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.stream;
|
||||
|
||||
let mut collected = Vec::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
collected.extend_from_slice(&chunk.unwrap().as_bytes());
|
||||
}
|
||||
|
||||
assert_eq!(collected, b"abcdef");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_chunk_stream_detects_hash_mismatch() {
|
||||
let shard_size = 4;
|
||||
let checksum_algo = HashAlgorithm::HighwayHash256S;
|
||||
let shard = b"abcd";
|
||||
|
||||
let mut encoded = Vec::new();
|
||||
let mut bad_hash = checksum_algo.hash_encode(shard).as_ref().to_vec();
|
||||
bad_hash[0] ^= 0xFF;
|
||||
encoded.extend_from_slice(&bad_hash);
|
||||
encoded.extend_from_slice(shard);
|
||||
|
||||
let result = create_bitrot_chunk_stream(
|
||||
Some(&encoded),
|
||||
None,
|
||||
"test-bucket",
|
||||
"test-path",
|
||||
0,
|
||||
shard.len(),
|
||||
shard.len(),
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut stream = result.unwrap().unwrap().stream;
|
||||
let err = stream.next().await.unwrap().unwrap_err();
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("bitrot hash mismatch"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_chunk_stream_trims_range_after_decode() {
|
||||
let shard_size = 4;
|
||||
let checksum_algo = HashAlgorithm::HighwayHash256S;
|
||||
let shard1 = b"abcd";
|
||||
let shard2 = b"efgh";
|
||||
|
||||
let mut encoded = Vec::new();
|
||||
encoded.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
|
||||
encoded.extend_from_slice(shard1);
|
||||
encoded.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
|
||||
encoded.extend_from_slice(shard2);
|
||||
|
||||
let mut stream = create_bitrot_chunk_stream(
|
||||
Some(&encoded),
|
||||
None,
|
||||
"test-bucket",
|
||||
"test-path",
|
||||
1,
|
||||
5,
|
||||
shard1.len() + shard2.len(),
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.stream;
|
||||
|
||||
let mut collected = Vec::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
collected.extend_from_slice(&chunk.unwrap().as_bytes());
|
||||
}
|
||||
|
||||
assert_eq!(collected, b"bcdef");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decode_bitrot_chunk_source_preserves_aligned_multi_chunk_slices() {
|
||||
let shard_size = 4;
|
||||
let checksum_algo = HashAlgorithm::Md5;
|
||||
let shard1 = b"abcd";
|
||||
let shard2 = b"efgh";
|
||||
|
||||
let mut encoded_chunk_one = Vec::new();
|
||||
encoded_chunk_one.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
|
||||
encoded_chunk_one.extend_from_slice(shard1);
|
||||
|
||||
let mut encoded_chunk_two = Vec::new();
|
||||
encoded_chunk_two.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
|
||||
encoded_chunk_two.extend_from_slice(shard2);
|
||||
|
||||
let source_chunks = vec![
|
||||
IoChunk::Shared(Bytes::from(encoded_chunk_one)),
|
||||
IoChunk::Shared(Bytes::from(encoded_chunk_two)),
|
||||
];
|
||||
let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
|
||||
|
||||
assert!(!copied, "frame-aligned multi-chunk source should not require aggregate copies");
|
||||
assert_eq!(decoded.len(), 2);
|
||||
assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
|
||||
assert_eq!(decoded[1].as_bytes(), Bytes::from_static(b"efgh"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decode_bitrot_chunk_source_marks_cross_chunk_frame_as_copied() {
|
||||
let shard_size = 4;
|
||||
let checksum_algo = HashAlgorithm::Md5;
|
||||
let shard1 = b"abcd";
|
||||
let shard2 = b"efgh";
|
||||
|
||||
let hash1 = checksum_algo.hash_encode(shard1).as_ref().to_vec();
|
||||
let hash2 = checksum_algo.hash_encode(shard2).as_ref().to_vec();
|
||||
let mut encoded = Vec::new();
|
||||
encoded.extend_from_slice(&hash1);
|
||||
encoded.extend_from_slice(shard1);
|
||||
encoded.extend_from_slice(&hash2);
|
||||
encoded.extend_from_slice(shard2);
|
||||
|
||||
let split = hash1.len() + 2;
|
||||
let source_chunks = vec![
|
||||
IoChunk::Shared(Bytes::copy_from_slice(&encoded[..split])),
|
||||
IoChunk::Shared(Bytes::copy_from_slice(&encoded[split..])),
|
||||
];
|
||||
let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
|
||||
|
||||
assert!(copied, "cross-chunk frame should be classified as requiring a copy");
|
||||
assert_eq!(decoded.len(), 2);
|
||||
assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
|
||||
assert_eq!(decoded[1].as_bytes(), Bytes::from_static(b"efgh"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decode_bitrot_chunk_source_preserves_pooled_single_chunk_slice() {
|
||||
let shard_size = 4;
|
||||
let checksum_algo = HashAlgorithm::Md5;
|
||||
let shard = b"abcd";
|
||||
|
||||
let mut encoded = Vec::new();
|
||||
encoded.extend_from_slice(checksum_algo.hash_encode(shard).as_ref());
|
||||
encoded.extend_from_slice(shard);
|
||||
|
||||
let source_chunks = vec![IoChunk::Pooled(rustfs_io_core::PooledChunk::from_vec(encoded))];
|
||||
let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
|
||||
|
||||
assert!(!copied, "single pooled chunk slice should preserve provenance without copy");
|
||||
assert_eq!(decoded.len(), 1);
|
||||
assert!(matches!(&decoded[0], IoChunk::Pooled(_)));
|
||||
assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bitrot_chunk_source_marks_cross_chunk_take_as_copied() {
|
||||
let source_stream: BoxChunkStream = Box::pin(stream::iter(vec![
|
||||
Ok(IoChunk::Shared(Bytes::from_static(b"ab"))),
|
||||
Ok(IoChunk::Shared(Bytes::from_static(b"cd"))),
|
||||
]));
|
||||
let mut source = BitrotChunkSource::new(source_stream, VecDeque::new(), false);
|
||||
|
||||
source.fill(4).await.expect("source fill should succeed");
|
||||
let span = source.take_span(4).expect("cross-chunk take should succeed");
|
||||
|
||||
assert!(span.copied, "cross-chunk take should be classified as copied");
|
||||
assert_eq!(span.bytes, Bytes::from_static(b"abcd"));
|
||||
assert_eq!(span.chunk.as_bytes(), Bytes::from_static(b"abcd"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bitrot_chunk_stream_state_yields_verified_prefix_before_later_truncation() {
|
||||
let shard_size = 4;
|
||||
let checksum_algo = HashAlgorithm::Md5;
|
||||
let shard1 = b"abcd";
|
||||
let shard2 = b"efgh";
|
||||
|
||||
let mut first_frame = Vec::new();
|
||||
first_frame.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
|
||||
first_frame.extend_from_slice(shard1);
|
||||
|
||||
let mut second_frame_prefix = Vec::new();
|
||||
second_frame_prefix.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
|
||||
second_frame_prefix.extend_from_slice(&shard2[..2]);
|
||||
|
||||
let source_stream: BoxChunkStream = Box::pin(stream::iter(vec![
|
||||
Ok(IoChunk::Shared(Bytes::from(first_frame))),
|
||||
Ok(IoChunk::Shared(Bytes::from(second_frame_prefix))),
|
||||
]));
|
||||
let mut state = BitrotChunkStreamState::new(
|
||||
source_stream,
|
||||
VecDeque::new(),
|
||||
false,
|
||||
shard1.len() + shard2.len(),
|
||||
0,
|
||||
shard1.len() + shard2.len(),
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
false,
|
||||
);
|
||||
|
||||
let first = state.next_verified_chunk().await.unwrap().unwrap();
|
||||
assert_eq!(first.as_bytes(), Bytes::from_static(b"abcd"));
|
||||
|
||||
let err = state.next_verified_chunk().await.unwrap_err();
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
|
||||
assert!(err.to_string().contains("truncated bitrot chunk source"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_reader_with_inline_offset_starts_at_requested_shard() {
|
||||
let shard_size = 4;
|
||||
|
||||
@@ -593,7 +593,7 @@ impl BucketTargetSys {
|
||||
};
|
||||
|
||||
if let Some(cli) = cli {
|
||||
return Some(cli.clone());
|
||||
return Some(cli);
|
||||
}
|
||||
|
||||
// TODO: spawn a task to reload the target
|
||||
@@ -786,7 +786,10 @@ impl BucketTargetSys {
|
||||
&& tgt
|
||||
.credentials
|
||||
.as_ref()
|
||||
.map(|c| c.access_key == target.credentials.as_ref().unwrap_or(&Credentials::default()).access_key)
|
||||
.map(|c| {
|
||||
let default_creds = Credentials::default();
|
||||
c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
|
||||
})
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return (tgt.arn.clone(), true);
|
||||
@@ -894,6 +897,41 @@ pub struct RemoveObjectOptions {
|
||||
pub replication_validity_check: bool,
|
||||
}
|
||||
|
||||
fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOptions) -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
if opts.force_delete {
|
||||
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
|
||||
}
|
||||
if opts.governance_bypass {
|
||||
headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
|
||||
}
|
||||
|
||||
if opts.replication_delete_marker {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
|
||||
}
|
||||
|
||||
if let Some(t) = opts.replication_mtime {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
|
||||
}
|
||||
|
||||
if !opts.replication_status.is_empty() {
|
||||
headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
|
||||
}
|
||||
|
||||
if let Some(version_id) = version_id {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, version_id);
|
||||
}
|
||||
|
||||
if opts.replication_request {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
}
|
||||
if opts.replication_validity_check {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
|
||||
}
|
||||
|
||||
headers
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AdvancedPutOptions {
|
||||
pub source_version_id: String,
|
||||
@@ -1424,39 +1462,15 @@ impl TargetClient {
|
||||
version_id: Option<String>,
|
||||
opts: RemoveObjectOptions,
|
||||
) -> Result<(), S3ClientError> {
|
||||
let mut headers = HeaderMap::new();
|
||||
if opts.force_delete {
|
||||
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
|
||||
}
|
||||
if opts.governance_bypass {
|
||||
headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
|
||||
}
|
||||
|
||||
if opts.replication_delete_marker {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
|
||||
}
|
||||
|
||||
if let Some(t) = opts.replication_mtime {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
|
||||
}
|
||||
|
||||
if !opts.replication_status.is_empty() {
|
||||
headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
|
||||
}
|
||||
|
||||
if opts.replication_request {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
}
|
||||
if opts.replication_validity_check {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
|
||||
}
|
||||
let headers = build_remove_object_headers(version_id.as_deref(), &opts);
|
||||
let api_version_id = if opts.replication_request { None } else { version_id };
|
||||
|
||||
match self
|
||||
.client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(version_id)
|
||||
.set_version_id(api_version_id)
|
||||
.customize()
|
||||
.map_request(move |mut req| {
|
||||
for (k, v) in headers.clone().into_iter() {
|
||||
@@ -1550,3 +1564,53 @@ impl From<std::io::Error> for BucketTargetError {
|
||||
}
|
||||
|
||||
impl Error for BucketTargetError {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn build_remove_object_headers_includes_internal_version_id_for_replication_delete() {
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
let headers = build_remove_object_headers(
|
||||
Some(version_id.as_str()),
|
||||
&RemoveObjectOptions {
|
||||
force_delete: false,
|
||||
governance_bypass: false,
|
||||
replication_delete_marker: true,
|
||||
replication_mtime: None,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
replication_request: true,
|
||||
replication_validity_check: false,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_VERSION_ID).as_deref(),
|
||||
Some(version_id.as_str()),
|
||||
"replication delete requests must preserve the version id in internal headers"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_remove_object_headers_omits_delete_marker_flag_for_marker_version_purge() {
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
let headers = build_remove_object_headers(
|
||||
Some(version_id.as_str()),
|
||||
&RemoveObjectOptions {
|
||||
force_delete: false,
|
||||
governance_bypass: false,
|
||||
replication_delete_marker: false,
|
||||
replication_mtime: None,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
replication_request: true,
|
||||
replication_validity_check: false,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_DELETEMARKER).is_none(),
|
||||
"delete-marker version purges must not masquerade as delete-marker creations"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+818
-263
File diff suppressed because it is too large
Load Diff
@@ -14,8 +14,9 @@
|
||||
|
||||
pub mod bucket_lifecycle_audit;
|
||||
pub mod bucket_lifecycle_ops;
|
||||
pub mod core;
|
||||
pub mod evaluator;
|
||||
pub mod lifecycle;
|
||||
pub use self::core as lifecycle;
|
||||
pub mod rule;
|
||||
pub mod tier_last_day_stats;
|
||||
pub mod tier_sweeper;
|
||||
|
||||
@@ -83,7 +83,7 @@ impl LastDayTierStats {
|
||||
#[allow(dead_code)]
|
||||
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
|
||||
let mut cl = self.clone();
|
||||
let mut cm = m.clone();
|
||||
let mut cm = m;
|
||||
let mut merged = LastDayTierStats::default();
|
||||
|
||||
if cl.updated_at.unix_timestamp() > cm.updated_at.unix_timestamp() {
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@@ -12,6 +11,7 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
@@ -109,10 +109,19 @@ impl ObjSweeper {
|
||||
}
|
||||
|
||||
pub async fn sweep(&self) {
|
||||
let je = self.should_remove_remote_object();
|
||||
if !je.is_none() {
|
||||
let mut expiry_state = GLOBAL_ExpiryState.write().await;
|
||||
expiry_state.enqueue_tier_journal_entry(&je.expect("err!"));
|
||||
let Some(je) = self.should_remove_remote_object() else {
|
||||
return;
|
||||
};
|
||||
let hash = je.op_hash();
|
||||
// Grab the sender under a short read lock, then release the lock so we
|
||||
// don't hold it across the async send.
|
||||
let wrkr = GLOBAL_ExpiryState.read().await.get_worker_ch(hash);
|
||||
let Some(wrkr) = wrkr else {
|
||||
GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
|
||||
return;
|
||||
};
|
||||
if wrkr.send(Some(Box::new(je))).await.is_err() {
|
||||
GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -678,7 +678,7 @@ impl BucketMetadata {
|
||||
// let x = data.clone();
|
||||
// let str = std::str::from_utf8(&x).expect("Invalid UTF-8");
|
||||
// println!("update config:{}", str);
|
||||
self.bucket_targets_config_json = data.clone();
|
||||
self.bucket_targets_config_json = data;
|
||||
self.bucket_targets_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_CORS_CONFIG => {
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use crate::bucket::metadata::BUCKET_METADATA_FILE;
|
||||
use crate::bucket::replication::{decode_resync_file, encode_resync_file};
|
||||
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::store_api::{BucketOptions, ChunkNativePutData, ObjectOptions, StorageAPI};
|
||||
use crate::store_api::{BucketOptions, ObjectOptions, PutObjReader, StorageAPI};
|
||||
use http::HeaderMap;
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use rustfs_policy::policy::PolicyDoc;
|
||||
@@ -263,7 +263,7 @@ async fn migrate_one_if_missing<S: StorageAPI>(
|
||||
}
|
||||
};
|
||||
|
||||
let mut put_data = ChunkNativePutData::from_vec(data);
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await {
|
||||
warn!("write {label}: {e}");
|
||||
} else {
|
||||
@@ -341,7 +341,7 @@ pub async fn try_migrate_iam_config<S: StorageAPI>(store: Arc<S>) {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mut put_data = ChunkNativePutData::from_vec(data);
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await {
|
||||
warn!("write IAM config {path}: {e}");
|
||||
} else {
|
||||
|
||||
@@ -152,7 +152,16 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
}
|
||||
|
||||
if obj.op_type == ReplicationType::Delete {
|
||||
if !rule.metadata_replicate(obj) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if obj.version_id.is_some() {
|
||||
if obj.delete_marker {
|
||||
return rule.delete_marker_replication.clone().is_some_and(|d| {
|
||||
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
|
||||
});
|
||||
}
|
||||
return rule
|
||||
.delete_replication
|
||||
.clone()
|
||||
|
||||
@@ -754,7 +754,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
||||
buckets: Vec<String>,
|
||||
) -> Result<(), EcstoreError> {
|
||||
// Load bucket metadata system in background
|
||||
let pool_clone = self.clone();
|
||||
let pool_clone = self;
|
||||
|
||||
tokio::spawn(async move {
|
||||
pool_clone.start_resync_routine(buckets, cancellation_token).await;
|
||||
@@ -824,10 +824,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
||||
let cancel_token = CancellationToken::new();
|
||||
resyncer.register_cancel_token(&opts, cancel_token.clone()).await;
|
||||
tokio::spawn(async move {
|
||||
resyncer
|
||||
.clone()
|
||||
.resync_bucket(cancel_token, storage, false, opts.clone())
|
||||
.await;
|
||||
Box::pin(resyncer.clone().resync_bucket(cancel_token, storage, false, opts.clone())).await;
|
||||
resyncer.clear_cancel_token(&opts).await;
|
||||
});
|
||||
|
||||
@@ -914,7 +911,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
||||
};
|
||||
tokio::spawn(async move {
|
||||
resync.register_cancel_token(&opts, ctx.clone()).await;
|
||||
resync.clone().resync_bucket(ctx, storage, true, opts.clone()).await;
|
||||
Box::pin(resync.clone().resync_bucket(ctx, storage, true, opts.clone())).await;
|
||||
resync.clear_cancel_token(&opts).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use crate::global::get_global_bucket_monitor;
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
|
||||
use crate::{StorageAPI, new_object_layer_fn};
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
|
||||
@@ -85,7 +85,8 @@ use tokio::task::JoinSet;
|
||||
use tokio::time::Duration as TokioDuration;
|
||||
use tokio_util::io::ReaderStream;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{error, info, instrument, warn};
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) const REPLICATION_DIR: &str = ".replication";
|
||||
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
|
||||
@@ -1271,15 +1272,7 @@ pub async fn check_replicate_delete(
|
||||
return ReplicateDecision::default();
|
||||
}
|
||||
|
||||
let opts = ObjectOpts {
|
||||
name: dobj.object_name.clone(),
|
||||
ssec: is_ssec_encrypted(&oi.user_defined),
|
||||
user_tags: oi.user_tags.clone(),
|
||||
delete_marker: oi.delete_marker,
|
||||
version_id: dobj.version_id,
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
let opts = delete_replication_object_opts(dobj, oi);
|
||||
|
||||
let tgt_arns = rcfg.filter_target_arns(&opts);
|
||||
let mut dsc = ReplicateDecision::new();
|
||||
@@ -1331,6 +1324,19 @@ pub async fn check_replicate_delete(
|
||||
dsc
|
||||
}
|
||||
|
||||
fn delete_replication_object_opts(dobj: &ObjectToDelete, oi: &ObjectInfo) -> ObjectOpts {
|
||||
ObjectOpts {
|
||||
name: dobj.object_name.clone(),
|
||||
ssec: is_ssec_encrypted(&oi.user_defined),
|
||||
user_tags: oi.user_tags.clone(),
|
||||
delete_marker: oi.delete_marker,
|
||||
version_id: dobj.version_id,
|
||||
op_type: ReplicationType::Delete,
|
||||
replica: oi.replication_status == ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the user-defined metadata contains SSEC encryption headers
|
||||
fn is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
||||
user_defined.contains_key(SSEC_ALGORITHM_HEADER)
|
||||
@@ -1497,6 +1503,54 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
|
||||
}
|
||||
};
|
||||
|
||||
if dobj.delete_object.delete_marker
|
||||
&& let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id
|
||||
{
|
||||
let source_marker_state = storage
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
&dobj.delete_object.object_name,
|
||||
&ObjectOptions {
|
||||
version_id: Some(delete_marker_version_id.to_string()),
|
||||
versioned: BucketVersioningSys::prefix_enabled(&bucket, &dobj.delete_object.object_name).await,
|
||||
version_suspended: BucketVersioningSys::prefix_suspended(&bucket, &dobj.delete_object.object_name).await,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
match source_marker_state {
|
||||
Ok(info) if info.delete_marker && info.version_id == Some(delete_marker_version_id) => {}
|
||||
Ok(_) => {
|
||||
warn!(
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = %delete_marker_version_id,
|
||||
"skipping stale delete-marker replication because source version is no longer a delete marker"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
warn!(
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = %delete_marker_version_id,
|
||||
"skipping stale delete-marker replication because source version no longer exists"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = %delete_marker_version_id,
|
||||
error = %err,
|
||||
"failed to verify source delete-marker state before replication"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let dsc = match parse_replicate_decision(
|
||||
&bucket,
|
||||
&dobj
|
||||
@@ -1529,7 +1583,6 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let ns_lock = match storage
|
||||
.new_ns_lock(&bucket, format!("/[replicate]/{}", dobj.delete_object.object_name).as_str())
|
||||
.await
|
||||
@@ -1653,7 +1706,33 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
|
||||
}
|
||||
}
|
||||
|
||||
let (replication_status, prev_status) = if dobj.delete_object.version_id.is_none() {
|
||||
let is_version_purge = is_version_delete_replication(&dobj.delete_object);
|
||||
|
||||
if should_retry_delete_marker_purge(&dobj.delete_object) {
|
||||
let bucket_clone = bucket.clone();
|
||||
let dobj_clone = dobj.clone();
|
||||
let dsc_clone = dsc.clone();
|
||||
let storage_clone = storage.clone();
|
||||
tokio::spawn(async move {
|
||||
for _ in 0..5 {
|
||||
if let Some(delete_marker_version_id) = dobj_clone.delete_object.delete_marker_version_id
|
||||
&& source_delete_marker_missing(
|
||||
&*storage_clone,
|
||||
&bucket_clone,
|
||||
&dobj_clone.delete_object.object_name,
|
||||
delete_marker_version_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(TokioDuration::from_secs(1)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let (replication_status, prev_status) = if !is_version_purge {
|
||||
(
|
||||
rinfos.replication_status(),
|
||||
dobj.delete_object
|
||||
@@ -1741,6 +1820,65 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
|
||||
}
|
||||
}
|
||||
|
||||
async fn source_delete_marker_missing<S: StorageAPI>(
|
||||
storage: &S,
|
||||
bucket: &str,
|
||||
object_name: &str,
|
||||
delete_marker_version_id: Uuid,
|
||||
) -> bool {
|
||||
match storage
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object_name,
|
||||
&ObjectOptions {
|
||||
version_id: Some(delete_marker_version_id.to_string()),
|
||||
versioned: BucketVersioningSys::prefix_enabled(bucket, object_name).await,
|
||||
version_suspended: BucketVersioningSys::prefix_suspended(bucket, object_name).await,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(info) => !info.delete_marker || info.version_id != Some(delete_marker_version_id),
|
||||
Err(err) => is_err_object_not_found(&err) || is_err_version_not_found(&err),
|
||||
}
|
||||
}
|
||||
|
||||
async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
|
||||
let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
|
||||
return;
|
||||
};
|
||||
|
||||
for tgt_entry in dsc.targets_map.values() {
|
||||
if !tgt_entry.replicate {
|
||||
continue;
|
||||
}
|
||||
if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
|
||||
continue;
|
||||
}
|
||||
let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &tgt_entry.arn).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let _ = tgt_client
|
||||
.remove_object(
|
||||
&tgt_client.bucket,
|
||||
&dobj.delete_object.object_name,
|
||||
Some(delete_marker_version_id.to_string()),
|
||||
RemoveObjectOptions {
|
||||
force_delete: false,
|
||||
governance_bypass: false,
|
||||
replication_delete_marker: false,
|
||||
replication_mtime: dobj.delete_object.delete_marker_mtime,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
replication_request: true,
|
||||
replication_validity_check: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
|
||||
let bucket = &dobj.bucket;
|
||||
let object_name = &dobj.delete_object.object_name;
|
||||
@@ -1924,6 +2062,18 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
|
||||
}
|
||||
}
|
||||
|
||||
fn is_version_delete_replication(dobj: &DeletedObject) -> bool {
|
||||
dobj.version_id.is_some() || (dobj.delete_marker_version_id.is_some() && !dobj.delete_marker)
|
||||
}
|
||||
|
||||
fn should_retry_delete_marker_purge(dobj: &DeletedObject) -> bool {
|
||||
dobj.delete_marker_version_id.is_some()
|
||||
}
|
||||
|
||||
fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Option<&str>) -> bool {
|
||||
!is_not_found && !matches!(code, Some("MethodNotAllowed" | "405"))
|
||||
}
|
||||
|
||||
async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
|
||||
let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
|
||||
version_id.to_owned()
|
||||
@@ -1941,7 +2091,8 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
rinfo.endpoint = tgt_client.endpoint.clone();
|
||||
rinfo.secure = tgt_client.secure;
|
||||
|
||||
if dobj.delete_object.version_id.is_none()
|
||||
let is_version_purge = is_version_delete_replication(&dobj.delete_object);
|
||||
if !is_version_purge
|
||||
&& rinfo.prev_replication_status == ReplicationStatusType::Completed
|
||||
&& dobj.op_type != ReplicationType::ExistingObject
|
||||
{
|
||||
@@ -1949,12 +2100,12 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
if dobj.delete_object.version_id.is_some() && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
|
||||
if is_version_purge && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
|
||||
if dobj.delete_object.version_id.is_none() {
|
||||
if !is_version_purge {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
} else {
|
||||
rinfo.version_purge_status = VersionPurgeStatusType::Failed;
|
||||
@@ -1968,18 +2119,29 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
Some(version_id.to_string())
|
||||
};
|
||||
|
||||
if dobj.delete_object.delete_marker_version_id.is_some()
|
||||
&& let Err(e) = tgt_client
|
||||
if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
match tgt_client
|
||||
.head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
|
||||
.await
|
||||
&& let SdkError::ServiceError(service_err) = &e
|
||||
&& !service_err.err().is_not_found()
|
||||
{
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
|
||||
return rinfo;
|
||||
};
|
||||
{
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
let non_retryable = matches!(
|
||||
&e,
|
||||
SdkError::ServiceError(service_err)
|
||||
if is_retryable_delete_replication_head_error(
|
||||
service_err.err().is_not_found(),
|
||||
service_err.err().code(),
|
||||
)
|
||||
);
|
||||
if non_retryable {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
return rinfo;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match tgt_client
|
||||
.remove_object(
|
||||
@@ -1989,7 +2151,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
RemoveObjectOptions {
|
||||
force_delete: false,
|
||||
governance_bypass: false,
|
||||
replication_delete_marker: dobj.delete_object.delete_marker_version_id.is_some(),
|
||||
replication_delete_marker: dobj.delete_object.delete_marker,
|
||||
replication_mtime: dobj.delete_object.delete_marker_mtime,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
replication_request: true,
|
||||
@@ -1999,15 +2161,32 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
if dobj.delete_object.version_id.is_none() {
|
||||
debug!(
|
||||
bucket = tgt_client.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = ?version_id,
|
||||
delete_marker = dobj.delete_object.delete_marker,
|
||||
is_version_purge,
|
||||
"replicate_delete_to_target succeeded"
|
||||
);
|
||||
if !is_version_purge {
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
} else {
|
||||
rinfo.version_purge_status = VersionPurgeStatusType::Complete;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
bucket = tgt_client.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = ?version_id,
|
||||
delete_marker = dobj.delete_object.delete_marker,
|
||||
is_version_purge,
|
||||
error = %e,
|
||||
"replicate_delete_to_target failed"
|
||||
);
|
||||
rinfo.error = Some(e.to_string());
|
||||
if dobj.delete_object.version_id.is_none() {
|
||||
if !is_version_purge {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
} else {
|
||||
rinfo.version_purge_status = VersionPurgeStatusType::Failed;
|
||||
@@ -3450,6 +3629,125 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_replication_object_opts_marks_replica_deletes() {
|
||||
let dobj = ObjectToDelete {
|
||||
object_name: "obj".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = ObjectInfo {
|
||||
bucket: "b".to_string(),
|
||||
name: "obj".to_string(),
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let opts = delete_replication_object_opts(&dobj, &oi);
|
||||
|
||||
assert!(
|
||||
opts.replica,
|
||||
"replica deletes must preserve replica status for downstream ReplicaModifications rules"
|
||||
);
|
||||
assert_eq!(opts.version_id, dobj.version_id);
|
||||
assert_eq!(opts.name, dobj.object_name);
|
||||
assert_eq!(opts.op_type, ReplicationType::Delete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_replication_object_opts_keeps_non_replica_deletes_local() {
|
||||
let dobj = ObjectToDelete {
|
||||
object_name: "obj".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = ObjectInfo {
|
||||
bucket: "b".to_string(),
|
||||
name: "obj".to_string(),
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let opts = delete_replication_object_opts(&dobj, &oi);
|
||||
|
||||
assert!(!opts.replica, "source-originated deletes should not be treated as replica modifications");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_version_delete_replication_for_delete_marker_version_purge() {
|
||||
let dobj = DeletedObject {
|
||||
delete_marker: false,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
is_version_delete_replication(&dobj),
|
||||
"delete-marker version purges must be tracked as version purge replication, not delete-marker creation replication"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_version_delete_replication_for_delete_marker_creation() {
|
||||
let dobj = DeletedObject {
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!is_version_delete_replication(&dobj),
|
||||
"delete-marker creation should remain on the delete-marker replication path"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_retry_delete_marker_purge_for_version_purge() {
|
||||
let dobj = DeletedObject {
|
||||
delete_marker: false,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
should_retry_delete_marker_purge(&dobj),
|
||||
"delete-marker version purge should schedule delayed target cleanup in case the target marker arrives late"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_retry_delete_marker_purge_for_delete_marker_creation() {
|
||||
let dobj = DeletedObject {
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
should_retry_delete_marker_purge(&dobj),
|
||||
"delete-marker creation should keep the late-arrival cleanup path so downstream purges can catch up"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
|
||||
assert!(
|
||||
!is_retryable_delete_replication_head_error(false, Some("405")),
|
||||
"numeric 405 responses should not block delete-marker purge replication"
|
||||
);
|
||||
assert!(
|
||||
!is_retryable_delete_replication_head_error(false, Some("MethodNotAllowed")),
|
||||
"MethodNotAllowed responses should not block delete-marker purge replication"
|
||||
);
|
||||
assert!(
|
||||
!is_retryable_delete_replication_head_error(true, Some("NoSuchKey")),
|
||||
"not-found responses should not block delete-marker purge replication"
|
||||
);
|
||||
assert!(
|
||||
is_retryable_delete_replication_head_error(false, Some("AccessDenied")),
|
||||
"unexpected head errors should still fail fast"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_heal_replicate_object_info_failed_object_returns_heal_roi() {
|
||||
let oi = ObjectInfo {
|
||||
|
||||
@@ -832,61 +832,55 @@ impl ReplicationStats {
|
||||
let mut rs = ReplStat::new();
|
||||
|
||||
match status {
|
||||
ReplicationStatusType::Pending => {
|
||||
if ri.op_type.is_data_replication() && prev_status != status {
|
||||
rs.set(
|
||||
ri.arn.clone(),
|
||||
ri.size,
|
||||
Duration::default(),
|
||||
status,
|
||||
ri.op_type,
|
||||
ri.endpoint.clone(),
|
||||
ri.secure,
|
||||
ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
|
||||
);
|
||||
}
|
||||
ReplicationStatusType::Pending if ri.op_type.is_data_replication() && prev_status != status => {
|
||||
rs.set(
|
||||
ri.arn.clone(),
|
||||
ri.size,
|
||||
Duration::default(),
|
||||
status,
|
||||
ri.op_type,
|
||||
ri.endpoint.clone(),
|
||||
ri.secure,
|
||||
ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
|
||||
);
|
||||
}
|
||||
ReplicationStatusType::Completed => {
|
||||
if ri.op_type.is_data_replication() {
|
||||
rs.set(
|
||||
ri.arn.clone(),
|
||||
ri.size,
|
||||
ri.duration,
|
||||
status,
|
||||
ri.op_type,
|
||||
ri.endpoint.clone(),
|
||||
ri.secure,
|
||||
ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
|
||||
);
|
||||
}
|
||||
ReplicationStatusType::Completed if ri.op_type.is_data_replication() => {
|
||||
rs.set(
|
||||
ri.arn.clone(),
|
||||
ri.size,
|
||||
ri.duration,
|
||||
status,
|
||||
ri.op_type,
|
||||
ri.endpoint.clone(),
|
||||
ri.secure,
|
||||
ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
|
||||
);
|
||||
}
|
||||
ReplicationStatusType::Failed => {
|
||||
if ri.op_type.is_data_replication() && prev_status == ReplicationStatusType::Pending {
|
||||
rs.set(
|
||||
ri.arn.clone(),
|
||||
ri.size,
|
||||
ri.duration,
|
||||
status,
|
||||
ri.op_type,
|
||||
ri.endpoint.clone(),
|
||||
ri.secure,
|
||||
ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
|
||||
);
|
||||
}
|
||||
ReplicationStatusType::Failed
|
||||
if ri.op_type.is_data_replication() && prev_status == ReplicationStatusType::Pending =>
|
||||
{
|
||||
rs.set(
|
||||
ri.arn.clone(),
|
||||
ri.size,
|
||||
ri.duration,
|
||||
status,
|
||||
ri.op_type,
|
||||
ri.endpoint.clone(),
|
||||
ri.secure,
|
||||
ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
|
||||
);
|
||||
}
|
||||
ReplicationStatusType::Replica => {
|
||||
if ri.op_type == ReplicationType::Object {
|
||||
rs.set(
|
||||
ri.arn.clone(),
|
||||
ri.size,
|
||||
Duration::default(),
|
||||
status,
|
||||
ri.op_type,
|
||||
String::new(),
|
||||
false,
|
||||
ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
|
||||
);
|
||||
}
|
||||
ReplicationStatusType::Replica if ri.op_type == ReplicationType::Object => {
|
||||
rs.set(
|
||||
ri.arn.clone(),
|
||||
ri.size,
|
||||
Duration::default(),
|
||||
status,
|
||||
ri.op_type,
|
||||
String::new(),
|
||||
false,
|
||||
ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
@@ -12,12 +12,16 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::disk_store::get_drive_walkdir_stall_timeout;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
||||
use std::{future::Future, pin::Pin};
|
||||
use std::{future::Future, pin::Pin, time::Duration};
|
||||
use tokio::io::AsyncRead;
|
||||
use tokio::spawn;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
@@ -26,6 +30,21 @@ pub type PartialFn =
|
||||
Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
|
||||
type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
|
||||
|
||||
#[derive(Debug)]
|
||||
enum PeekOutcome {
|
||||
Ready(Option<MetaCacheEntry>),
|
||||
Error(rustfs_filemeta::Error),
|
||||
TimedOut,
|
||||
}
|
||||
|
||||
async fn peek_with_timeout<R: AsyncRead + Unpin>(reader: &mut MetacacheReader<R>, timeout_duration: Duration) -> PeekOutcome {
|
||||
match timeout(timeout_duration, reader.peek()).await {
|
||||
Ok(Ok(entry)) => PeekOutcome::Ready(entry),
|
||||
Ok(Err(err)) => PeekOutcome::Error(err),
|
||||
Err(_) => PeekOutcome::TimedOut,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ListPathRawOptions {
|
||||
pub disks: Vec<Option<DiskStore>>,
|
||||
@@ -160,6 +179,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
}
|
||||
|
||||
let revjob = spawn(async move {
|
||||
let peek_timeout = get_drive_walkdir_stall_timeout();
|
||||
let mut errs: Vec<Option<DiskError>> = Vec::with_capacity(readers.len());
|
||||
for _ in 0..readers.len() {
|
||||
errs.push(None);
|
||||
@@ -191,8 +211,8 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
continue;
|
||||
}
|
||||
|
||||
let entry = match r.peek().await {
|
||||
Ok(res) => {
|
||||
let entry = match peek_with_timeout(r, peek_timeout).await {
|
||||
PeekOutcome::Ready(res) => {
|
||||
if let Some(entry) = res {
|
||||
// info!("read entry disk: {}, name: {}", i, entry.name);
|
||||
entry
|
||||
@@ -203,7 +223,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
PeekOutcome::Error(err) => {
|
||||
if err == rustfs_filemeta::Error::Unexpected {
|
||||
at_eof += 1;
|
||||
// warn!("list_path_raw: peek err eof, disk: {}", i);
|
||||
@@ -236,6 +256,31 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
continue;
|
||||
}
|
||||
}
|
||||
PeekOutcome::TimedOut => {
|
||||
has_err += 1;
|
||||
errs[i] = Some(DiskError::Timeout);
|
||||
let endpoint = opts
|
||||
.disks
|
||||
.get(i)
|
||||
.and_then(|disk| disk.as_ref().map(|disk| disk.endpoint().to_string()))
|
||||
.unwrap_or_else(|| "missing".to_string());
|
||||
counter!(
|
||||
"rustfs_list_path_raw_stall_total",
|
||||
"drive" => endpoint.clone()
|
||||
)
|
||||
.increment(1);
|
||||
warn!(
|
||||
drive = %endpoint,
|
||||
bucket = %opts.bucket,
|
||||
path = %opts.path,
|
||||
timeout_ms = peek_timeout.as_millis(),
|
||||
"list_path_raw reader peek timed out; excluding drive from current merge"
|
||||
);
|
||||
let (detached_rd, write_half) = tokio::io::duplex(1);
|
||||
drop(write_half);
|
||||
*r = MetacacheReader::new(detached_rd);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
// warn!("list_path_raw: loop entry: {:?}, disk: {}", &entry.name, i);
|
||||
@@ -364,3 +409,42 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
// warn!("list_path_raw: done");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_filemeta::MetacacheWriter;
|
||||
|
||||
#[tokio::test]
|
||||
async fn peek_with_timeout_times_out_on_silent_reader() {
|
||||
let (_writer, reader) = tokio::io::duplex(64);
|
||||
let mut reader = MetacacheReader::new(reader);
|
||||
|
||||
let outcome = peek_with_timeout(&mut reader, Duration::from_millis(20)).await;
|
||||
assert!(matches!(outcome, PeekOutcome::TimedOut));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peek_with_timeout_reads_entry_before_deadline() {
|
||||
let (reader, writer) = tokio::io::duplex(256);
|
||||
let mut metacache_reader = MetacacheReader::new(reader);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut writer = MetacacheWriter::new(writer);
|
||||
let entry = MetaCacheEntry {
|
||||
name: "bucket/object".to_string(),
|
||||
metadata: vec![1, 2, 3],
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
writer.write(&[entry]).await.expect("entry should be written");
|
||||
writer.close().await.expect("writer should close");
|
||||
});
|
||||
|
||||
let outcome = peek_with_timeout(&mut metacache_reader, Duration::from_secs(1)).await;
|
||||
match outcome {
|
||||
PeekOutcome::Ready(Some(entry)) => assert_eq!(entry.name, "bucket/object"),
|
||||
other => panic!("expected ready entry, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,7 +100,7 @@ pub fn http_resp_to_error_response(
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
) -> ErrorResponse {
|
||||
let err_body = String::from_utf8(b).unwrap();
|
||||
let err_body = String::from_utf8_lossy(&b).to_string();
|
||||
if h.is_empty() || resp_status.is_client_error() || resp_status.is_server_error() {
|
||||
return ErrorResponse {
|
||||
status_code: resp_status,
|
||||
@@ -178,36 +178,46 @@ pub fn http_resp_to_error_response(
|
||||
};
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err_resp = err_resp_.unwrap();
|
||||
} else if let Ok(parsed_resp) = err_resp_ {
|
||||
err_resp = parsed_resp;
|
||||
}
|
||||
err_resp.status_code = resp_status;
|
||||
if let Some(server_name) = h.get("Server") {
|
||||
err_resp.server = server_name.to_str().expect("err").to_string();
|
||||
if let Ok(server_str) = server_name.to_str() {
|
||||
err_resp.server = server_str.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
let code = h.get("x-minio-error-code");
|
||||
if code.is_some() {
|
||||
err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
|
||||
if let Some(code) = h.get("x-minio-error-code") {
|
||||
if let Ok(code_str) = code.to_str() {
|
||||
err_resp.code = S3ErrorCode::Custom(code_str.into());
|
||||
}
|
||||
}
|
||||
let desc = h.get("x-minio-error-desc");
|
||||
if desc.is_some() {
|
||||
err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
|
||||
if let Some(desc) = h.get("x-minio-error-desc") {
|
||||
if let Ok(desc_str) = desc.to_str() {
|
||||
err_resp.message = desc_str.trim_matches('"').to_string();
|
||||
}
|
||||
}
|
||||
|
||||
if err_resp.request_id == "" {
|
||||
if let Some(x_amz_request_id) = h.get("x-amz-request-id") {
|
||||
err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
|
||||
if let Ok(request_id_str) = x_amz_request_id.to_str() {
|
||||
err_resp.request_id = request_id_str.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
if err_resp.host_id == "" {
|
||||
if let Some(x_amz_id_2) = h.get("x-amz-id-2") {
|
||||
err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
|
||||
if let Ok(host_id_str) = x_amz_id_2.to_str() {
|
||||
err_resp.host_id = host_id_str.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
if err_resp.region == "" {
|
||||
if let Some(x_amz_bucket_region) = h.get("x-amz-bucket-region") {
|
||||
err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
|
||||
if let Ok(region_str) = x_amz_bucket_region.to_str() {
|
||||
err_resp.region = region_str.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@@ -12,6 +11,7 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(clippy::map_entry)]
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
|
||||
@@ -137,21 +137,21 @@ impl Default for PutObjectOptions {
|
||||
impl PutObjectOptions {
|
||||
fn set_match_etag(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
.insert("If-Match", HeaderValue::from_str("*").expect("err"));
|
||||
self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
|
||||
} else {
|
||||
self.custom_header
|
||||
.insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
|
||||
if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{}\"", etag)) {
|
||||
self.custom_header.insert("If-Match", etag_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn set_match_etag_except(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
.insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
|
||||
self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
|
||||
} else {
|
||||
self.custom_header
|
||||
.insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
|
||||
if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{etag}\"")) {
|
||||
self.custom_header.insert("If-None-Match", etag_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,59 +162,75 @@ impl PutObjectOptions {
|
||||
if content_type == "" {
|
||||
content_type = "application/octet-stream".to_string();
|
||||
}
|
||||
header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err"));
|
||||
if let Ok(content_type_value) = HeaderValue::from_str(&content_type) {
|
||||
header.insert("Content-Type", content_type_value);
|
||||
}
|
||||
|
||||
if self.content_encoding != "" {
|
||||
header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err"));
|
||||
if let Ok(encoding_value) = HeaderValue::from_str(&self.content_encoding) {
|
||||
header.insert("Content-Encoding", encoding_value);
|
||||
}
|
||||
}
|
||||
if self.content_disposition != "" {
|
||||
header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err"));
|
||||
if let Ok(disposition_value) = HeaderValue::from_str(&self.content_disposition) {
|
||||
header.insert("Content-Disposition", disposition_value);
|
||||
}
|
||||
}
|
||||
if self.content_language != "" {
|
||||
header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err"));
|
||||
if let Ok(language_value) = HeaderValue::from_str(&self.content_language) {
|
||||
header.insert("Content-Language", language_value);
|
||||
}
|
||||
}
|
||||
if self.cache_control != "" {
|
||||
header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err"));
|
||||
if let Ok(cache_value) = HeaderValue::from_str(&self.cache_control) {
|
||||
header.insert("Cache-Control", cache_value);
|
||||
}
|
||||
}
|
||||
|
||||
if self.expires.unix_timestamp() != 0 {
|
||||
header.insert(
|
||||
"Expires",
|
||||
HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
|
||||
); //rustfs invalid header
|
||||
if let Ok(expires_str) = self.expires.format(ISO8601_DATEFORMAT) {
|
||||
if let Ok(expires_value) = HeaderValue::from_str(&expires_str) {
|
||||
header.insert("Expires", expires_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if self.mode.as_str() != "" {
|
||||
header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err"));
|
||||
if let Ok(mode_value) = HeaderValue::from_str(self.mode.as_str()) {
|
||||
header.insert(X_AMZ_OBJECT_LOCK_MODE, mode_value);
|
||||
}
|
||||
}
|
||||
|
||||
if self.retain_until_date.unix_timestamp() != 0 {
|
||||
header.insert(
|
||||
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
|
||||
HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
|
||||
);
|
||||
if let Ok(retain_str) = self.retain_until_date.format(ISO8601_DATEFORMAT) {
|
||||
if let Ok(retain_value) = HeaderValue::from_str(&retain_str) {
|
||||
header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, retain_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if self.legalhold.as_str() != "" {
|
||||
header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err"));
|
||||
if let Ok(legalhold_value) = HeaderValue::from_str(self.legalhold.as_str()) {
|
||||
header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, legalhold_value);
|
||||
}
|
||||
}
|
||||
|
||||
if self.storage_class != "" {
|
||||
header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err"));
|
||||
if let Ok(storage_class_value) = HeaderValue::from_str(&self.storage_class) {
|
||||
header.insert(X_AMZ_STORAGE_CLASS, storage_class_value);
|
||||
}
|
||||
}
|
||||
|
||||
if self.website_redirect_location != "" {
|
||||
header.insert(
|
||||
X_AMZ_WEBSITE_REDIRECT_LOCATION,
|
||||
HeaderValue::from_str(&self.website_redirect_location).expect("err"),
|
||||
);
|
||||
if let Ok(redirect_value) = HeaderValue::from_str(&self.website_redirect_location) {
|
||||
header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, redirect_value);
|
||||
}
|
||||
}
|
||||
|
||||
if !self.internal.replication_status.as_str().is_empty() {
|
||||
header.insert(
|
||||
X_AMZ_REPLICATION_STATUS,
|
||||
HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"),
|
||||
);
|
||||
if let Ok(replication_status_value) = HeaderValue::from_str(self.internal.replication_status.as_str()) {
|
||||
header.insert(X_AMZ_REPLICATION_STATUS, replication_status_value);
|
||||
}
|
||||
}
|
||||
|
||||
for (k, v) in &self.user_metadata {
|
||||
@@ -360,17 +376,21 @@ impl TransitionClient {
|
||||
|
||||
let mut md5_base64: String = "".to_string();
|
||||
if opts.send_content_md5 {
|
||||
let mut md5_hasher = self.md5_hasher.lock().unwrap();
|
||||
let hash = md5_hasher.as_mut().expect("err");
|
||||
let hash = hash.hash_encode(&buf[..length]);
|
||||
md5_base64 = base64_encode(hash.as_ref());
|
||||
if let Some(mut md5_hasher) = self.md5_hasher.lock().unwrap().as_mut() {
|
||||
let hash = md5_hasher.hash_encode(&buf[..length]);
|
||||
md5_base64 = base64_encode(hash.as_ref());
|
||||
}
|
||||
} else {
|
||||
let mut crc = opts.auto_checksum.hasher()?;
|
||||
crc.update(&buf[..length]);
|
||||
let csum = crc.finalize();
|
||||
|
||||
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
|
||||
custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
|
||||
if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
|
||||
custom_header.insert(header_name, header_value);
|
||||
} else {
|
||||
warn!("Failed to parse checksum value");
|
||||
}
|
||||
} else {
|
||||
warn!("Invalid header name: {}", opts.auto_checksum.key());
|
||||
}
|
||||
|
||||
@@ -127,7 +127,11 @@ impl TransitionClient {
|
||||
let csum = crc.finalize();
|
||||
|
||||
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
|
||||
custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
|
||||
if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
|
||||
custom_header.insert(header_name, header_value);
|
||||
} else {
|
||||
warn!("Failed to parse checksum value");
|
||||
}
|
||||
} else {
|
||||
warn!("Invalid header name: {}", opts.auto_checksum.key());
|
||||
}
|
||||
@@ -309,27 +313,27 @@ impl TransitionClient {
|
||||
let h = resp.headers();
|
||||
let mut obj_part = ObjectPart {
|
||||
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
|
||||
h_checksum_crc32.to_str().expect("err").to_string()
|
||||
h_checksum_crc32.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
|
||||
h_checksum_crc32c.to_str().expect("err").to_string()
|
||||
h_checksum_crc32c.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
|
||||
h_checksum_sha1.to_str().expect("err").to_string()
|
||||
h_checksum_sha1.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
|
||||
h_checksum_sha256.to_str().expect("err").to_string()
|
||||
h_checksum_sha256.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
|
||||
h_checksum_crc64nvme.to_str().expect("err").to_string()
|
||||
h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
@@ -338,7 +342,7 @@ impl TransitionClient {
|
||||
obj_part.size = p.size;
|
||||
obj_part.part_num = p.part_number;
|
||||
obj_part.etag = if let Some(h_etag) = h.get("ETag") {
|
||||
h_etag.to_str().expect("err").trim_matches('"').to_string()
|
||||
h_etag.to_str().unwrap_or("").trim_matches('"').to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
};
|
||||
@@ -398,7 +402,7 @@ impl TransitionClient {
|
||||
key: complete_multipart_upload_result.key,
|
||||
etag: trim_etag(&complete_multipart_upload_result.etag),
|
||||
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
|
||||
h_x_amz_version_id.to_str().expect("err").to_string()
|
||||
h_x_amz_version_id.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
use bytes::Bytes;
|
||||
use futures::future::join_all;
|
||||
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
|
||||
use std::io::Error;
|
||||
use std::sync::RwLock;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use time::{OffsetDateTime, format_description};
|
||||
@@ -152,7 +153,10 @@ impl TransitionClient {
|
||||
|
||||
if opts.send_content_md5 {
|
||||
let mut md5_hasher = self.md5_hasher.lock().unwrap();
|
||||
let md5_hash = md5_hasher.as_mut().expect("err");
|
||||
let md5_hash = match md5_hasher.as_mut() {
|
||||
Some(hasher) => hasher,
|
||||
None => return Err(std::io::Error::other("MD5 hasher not initialized")),
|
||||
};
|
||||
let hash = md5_hash.hash_encode(&buf[..length]);
|
||||
md5_base64 = base64_encode(hash.as_ref());
|
||||
} else {
|
||||
@@ -161,7 +165,11 @@ impl TransitionClient {
|
||||
let csum = crc.finalize();
|
||||
|
||||
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
|
||||
custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
|
||||
if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
|
||||
custom_header.insert(header_name, header_value);
|
||||
} else {
|
||||
warn!("Failed to parse checksum value");
|
||||
}
|
||||
} else {
|
||||
warn!("Invalid header name: {}", opts.auto_checksum.key());
|
||||
}
|
||||
@@ -275,11 +283,14 @@ impl TransitionClient {
|
||||
for part_number in 1..=total_parts_count {
|
||||
let mut buf = Vec::<u8>::new();
|
||||
select! {
|
||||
buf = bufs_rx.recv() => {}
|
||||
buf1 = bufs_rx.recv() => {
|
||||
if let Some(buf1) = buf1 {
|
||||
buf = buf1;
|
||||
}
|
||||
}
|
||||
err = err_rx.recv() => {
|
||||
//cancel_token.cancel();
|
||||
//wg.Wait()
|
||||
return Err(err.expect("err"));
|
||||
return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
|
||||
}
|
||||
else => (),
|
||||
}
|
||||
@@ -309,7 +320,11 @@ impl TransitionClient {
|
||||
let csum = crc.finalize();
|
||||
|
||||
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
|
||||
custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
|
||||
if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
|
||||
custom_header.insert(header_name, header_value);
|
||||
} else {
|
||||
warn!("Failed to parse checksum value");
|
||||
}
|
||||
} else {
|
||||
warn!("Invalid header name: {}", opts.auto_checksum.key());
|
||||
}
|
||||
@@ -319,12 +334,19 @@ impl TransitionClient {
|
||||
let clone_parts_info = parts_info.clone();
|
||||
let clone_upload_id = upload_id.clone();
|
||||
let clone_self = self.clone();
|
||||
let err_tx_clone = err_tx.clone();
|
||||
futures.push(async move {
|
||||
let mut md5_base64: String = "".to_string();
|
||||
|
||||
if opts.send_content_md5 {
|
||||
let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
|
||||
let md5_hash = md5_hasher.as_mut().expect("err");
|
||||
let md5_hash = match md5_hasher.as_mut() {
|
||||
Some(hasher) => hasher,
|
||||
None => {
|
||||
//let _ = err_tx_clone.send(std::io::Error::other("MD5 hasher not initialized")).await;
|
||||
return Ok::<(), Error>(());
|
||||
}
|
||||
};
|
||||
let hash = md5_hash.hash_encode(&buf[..length]);
|
||||
md5_base64 = base64_encode(hash.as_ref());
|
||||
}
|
||||
@@ -344,12 +366,21 @@ impl TransitionClient {
|
||||
sha256_hex: "".to_string(),
|
||||
trailer: HeaderMap::new(),
|
||||
};
|
||||
let obj_part = clone_self.upload_part(&mut p).await.expect("err");
|
||||
let obj_part = match clone_self.upload_part(&mut p).await {
|
||||
Ok(part) => part,
|
||||
Err(err) => {
|
||||
let _ = err_tx_clone.send(std::io::Error::other(err.to_string())).await;
|
||||
return Err::<(), Error>(err);
|
||||
}
|
||||
};
|
||||
|
||||
let mut clone_parts_info = clone_parts_info.write().unwrap();
|
||||
clone_parts_info.entry(part_number).or_insert(obj_part);
|
||||
{
|
||||
let mut clone_parts_info = clone_parts_info.write().unwrap();
|
||||
clone_parts_info.entry(part_number).or_insert(obj_part);
|
||||
}
|
||||
|
||||
clone_bufs_tx.send(buf);
|
||||
let _ = clone_bufs_tx.send(buf).await;
|
||||
Ok::<(), Error>(())
|
||||
});
|
||||
|
||||
total_uploaded_size += length as i64;
|
||||
@@ -359,7 +390,7 @@ impl TransitionClient {
|
||||
|
||||
select! {
|
||||
err = err_rx.recv() => {
|
||||
return Err(err.expect("err"));
|
||||
return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
|
||||
}
|
||||
else => (),
|
||||
}
|
||||
@@ -504,9 +535,10 @@ impl TransitionClient {
|
||||
Ok(UploadInfo {
|
||||
bucket: bucket_name.to_string(),
|
||||
key: object_name.to_string(),
|
||||
etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
|
||||
etag: trim_etag(h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("")),
|
||||
|
||||
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
|
||||
h_x_amz_version_id.to_str().expect("err").to_string()
|
||||
h_x_amz_version_id.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
@@ -514,27 +546,27 @@ impl TransitionClient {
|
||||
expiration: exp_time,
|
||||
expiration_rule_id: rule_id,
|
||||
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
|
||||
h_checksum_crc32.to_str().expect("err").to_string()
|
||||
h_checksum_crc32.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
|
||||
h_checksum_crc32c.to_str().expect("err").to_string()
|
||||
h_checksum_crc32c.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
|
||||
h_checksum_sha1.to_str().expect("err").to_string()
|
||||
h_checksum_sha1.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
|
||||
h_checksum_sha256.to_str().expect("err").to_string()
|
||||
h_checksum_sha256.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
|
||||
h_checksum_crc64nvme.to_str().expect("err").to_string()
|
||||
h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
|
||||
@@ -25,7 +25,7 @@ use hyper::body::Bytes;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::ReplicationStatus;
|
||||
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
|
||||
use s3s::header::{X_AMZ_BYPASS_GOVERNANCE_RETENTION, X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
|
||||
use serde::Deserialize;
|
||||
use std::fmt::Display;
|
||||
use std::{
|
||||
@@ -111,8 +111,9 @@ impl TransitionClient {
|
||||
.await?;
|
||||
|
||||
{
|
||||
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
|
||||
bucket_loc_cache.delete(bucket_name);
|
||||
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
|
||||
bucket_loc_cache.delete(bucket_name);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -142,8 +143,9 @@ impl TransitionClient {
|
||||
.await?;
|
||||
|
||||
{
|
||||
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
|
||||
bucket_loc_cache.delete(bucket_name);
|
||||
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
|
||||
bucket_loc_cache.delete(bucket_name);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -168,7 +170,7 @@ impl TransitionClient {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
if opts.governance_bypass {
|
||||
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance
|
||||
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true")); //amzBypassGovernance
|
||||
}
|
||||
|
||||
let resp = self
|
||||
@@ -197,13 +199,12 @@ impl TransitionClient {
|
||||
Ok(RemoveObjectResult {
|
||||
object_name: object_name.to_string(),
|
||||
object_version_id: opts.version_id,
|
||||
delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
|
||||
delete_marker: resp.headers().get(X_AMZ_DELETE_MARKER).map_or(false, |v| v == "true"),
|
||||
delete_marker_version_id: resp
|
||||
.headers()
|
||||
.get("x-amz-version-id")
|
||||
.expect("err")
|
||||
.to_str()
|
||||
.expect("err")
|
||||
.get(X_AMZ_VERSION_ID)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
@@ -290,15 +291,15 @@ impl TransitionClient {
|
||||
bucket_name,
|
||||
&object.name,
|
||||
RemoveObjectOptions {
|
||||
version_id: object.version_id.expect("err").to_string(),
|
||||
version_id: object.version_id.map(|id| id.to_string()).unwrap_or_default(),
|
||||
governance_bypass: opts.governance_bypass,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let remove_result_clone = remove_result.clone();
|
||||
if !remove_result.err.is_none() {
|
||||
match to_error_response(&remove_result.err.expect("err")).code {
|
||||
if let Some(err) = &remove_result.err {
|
||||
match to_error_response(err).code {
|
||||
S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
|
||||
continue;
|
||||
}
|
||||
@@ -326,7 +327,7 @@ impl TransitionClient {
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
if opts.governance_bypass {
|
||||
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));
|
||||
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true"));
|
||||
}
|
||||
|
||||
let remove_bytes = generate_remove_multi_objects_request(&batch);
|
||||
@@ -423,23 +424,20 @@ impl TransitionClient {
|
||||
request_id: resp
|
||||
.headers()
|
||||
.get("x-amz-request-id")
|
||||
.expect("err")
|
||||
.to_str()
|
||||
.expect("err")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string(),
|
||||
host_id: resp
|
||||
.headers()
|
||||
.get("x-amz-id-2")
|
||||
.expect("err")
|
||||
.to_str()
|
||||
.expect("err")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string(),
|
||||
region: resp
|
||||
.headers()
|
||||
.get("x-amz-bucket-region")
|
||||
.expect("err")
|
||||
.to_str()
|
||||
.expect("err")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -472,10 +470,11 @@ pub struct RemoveObjectError {
|
||||
|
||||
impl Display for RemoveObjectError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
if self.err.is_none() {
|
||||
return write!(f, "unexpected remove object error result");
|
||||
if let Some(err) = &self.err {
|
||||
write!(f, "{}", err.to_string())
|
||||
} else {
|
||||
write!(f, "unexpected remove object error result")
|
||||
}
|
||||
write!(f, "{}", self.err.as_ref().expect("err").to_string())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -70,10 +70,10 @@ impl TransitionClient {
|
||||
|
||||
let mut location;
|
||||
{
|
||||
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
|
||||
let ret = bucket_loc_cache.get(bucket_name);
|
||||
if let Some(location) = ret {
|
||||
return Ok(location);
|
||||
if let Ok(bucket_loc_cache) = self.bucket_loc_cache.lock() {
|
||||
if let Some(location) = bucket_loc_cache.get(bucket_name) {
|
||||
return Ok(location);
|
||||
}
|
||||
}
|
||||
//location = ret?;
|
||||
}
|
||||
@@ -83,8 +83,9 @@ impl TransitionClient {
|
||||
let mut resp = self.doit(req).await?;
|
||||
location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
|
||||
{
|
||||
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
|
||||
bucket_loc_cache.set(bucket_name, &location);
|
||||
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
|
||||
bucket_loc_cache.set(bucket_name, &location);
|
||||
}
|
||||
}
|
||||
Ok(location)
|
||||
}
|
||||
@@ -108,7 +109,11 @@ impl TransitionClient {
|
||||
url_str.push_str("://");
|
||||
url_str.push_str(bucket_name);
|
||||
url_str.push_str(".");
|
||||
url_str.push_str(target_url.host_str().expect("err"));
|
||||
url_str.push_str(
|
||||
target_url
|
||||
.host_str()
|
||||
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "host is none"))?,
|
||||
);
|
||||
url_str.push_str("/?location");
|
||||
} else {
|
||||
let mut path = bucket_name.to_string();
|
||||
@@ -135,13 +140,16 @@ impl TransitionClient {
|
||||
|
||||
let value;
|
||||
{
|
||||
let mut creds_provider = self.creds_provider.lock().unwrap();
|
||||
value = match creds_provider.get_with_context(Some(self.cred_context())) {
|
||||
Ok(v) => v,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
};
|
||||
if let Ok(mut creds_provider) = self.creds_provider.lock() {
|
||||
value = match creds_provider.get_with_context(Some(self.cred_context())) {
|
||||
Ok(v) => v,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
};
|
||||
} else {
|
||||
return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
|
||||
}
|
||||
}
|
||||
|
||||
let mut signer_type = value.signer_type.clone();
|
||||
@@ -171,8 +179,9 @@ impl TransitionClient {
|
||||
content_sha256 = UNSIGNED_PAYLOAD.to_string();
|
||||
}
|
||||
|
||||
req.headers_mut()
|
||||
.insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
|
||||
if let Ok(content_sha256_value) = content_sha256.parse() {
|
||||
req.headers_mut().insert("X-Amz-Content-Sha256", content_sha256_value);
|
||||
}
|
||||
let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
|
||||
Ok(req)
|
||||
}
|
||||
@@ -228,13 +237,16 @@ async fn process_bucket_location_response(
|
||||
}
|
||||
let mut location = "".to_string();
|
||||
if tier_type == "huaweicloud" {
|
||||
let d = quick_xml::de::from_str::<CreateBucketConfiguration>(&String::from_utf8(body_vec).unwrap()).unwrap();
|
||||
location = d.location_constraint;
|
||||
if let Ok(body_str) = String::from_utf8(body_vec) {
|
||||
if let Ok(d) = quick_xml::de::from_str::<CreateBucketConfiguration>(&body_str) {
|
||||
location = d.location_constraint;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if let Ok(LocationConstraint { field }) =
|
||||
quick_xml::de::from_str::<LocationConstraint>(&String::from_utf8(body_vec).unwrap())
|
||||
{
|
||||
location = field;
|
||||
if let Ok(body_str) = String::from_utf8(body_vec) {
|
||||
if let Ok(LocationConstraint { field }) = quick_xml::de::from_str::<LocationConstraint>(&body_str) {
|
||||
location = field;
|
||||
}
|
||||
}
|
||||
}
|
||||
//debug!("location: {}", location);
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@@ -12,6 +11,7 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
@@ -57,7 +57,9 @@ impl<P: Provider + Default> Credentials<P> {
|
||||
|
||||
pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
|
||||
if self.is_expired() {
|
||||
let creds = self.provider.retrieve_with_cred_context(cc.expect("err"));
|
||||
let creds = self.provider.retrieve_with_cred_context(cc.unwrap_or(CredContext {
|
||||
endpoint: "".to_string(),
|
||||
}));
|
||||
self.creds = creds;
|
||||
self.force_refresh = false;
|
||||
}
|
||||
|
||||
@@ -12,14 +12,38 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::bucket::lifecycle::lifecycle;
|
||||
use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_delete, schedule_replication_delete};
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete};
|
||||
use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
|
||||
use rustfs_lock::MAX_DELETE_LIST;
|
||||
|
||||
pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
|
||||
fn lifecycle_version_delete_replication_state(
|
||||
replicate_decision_str: String,
|
||||
pending_status: Option<String>,
|
||||
) -> ReplicationState {
|
||||
ReplicationState {
|
||||
replicate_decision_str,
|
||||
version_purge_status_internal: pending_status.clone(),
|
||||
purge_targets: version_purge_statuses_map(pending_status.as_deref().unwrap_or_default()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
|
||||
let version_suspended = match BucketVersioningSys::get(bucket).await {
|
||||
Ok(vc) => vc.suspended(),
|
||||
Err(err) => {
|
||||
warn!(bucket, error = ?err, "failed to get versioning config during lifecycle noncurrent version cleanup");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let mut remaining = to_del;
|
||||
loop {
|
||||
let mut to_del = remaining;
|
||||
@@ -29,15 +53,94 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object
|
||||
} else {
|
||||
remaining = &[];
|
||||
}
|
||||
let vc = BucketVersioningSys::get(bucket).await.expect("err!");
|
||||
let _deleted_objs = api.delete_objects(
|
||||
bucket,
|
||||
to_del.to_vec(),
|
||||
ObjectOptions {
|
||||
//prefix_enabled_fn: vc.prefix_enabled(""),
|
||||
version_suspended: vc.suspended(),
|
||||
|
||||
let mut replication_candidates: Vec<Option<ReplicationState>> = Vec::with_capacity(to_del.len());
|
||||
for object in to_del.iter() {
|
||||
let version_id = object.version_id.map(|vid| vid.to_string());
|
||||
let opts = ObjectOptions {
|
||||
version_id: version_id.clone(),
|
||||
versioned: true,
|
||||
version_suspended,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
};
|
||||
let candidate = match api.get_object_info(bucket, &object.object_name, &opts).await {
|
||||
Ok(info) => {
|
||||
let dsc = check_replicate_delete(bucket, object, &info, &opts, None).await;
|
||||
dsc.replicate_any()
|
||||
.then(|| lifecycle_version_delete_replication_state(dsc.to_string(), dsc.pending_status()))
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
bucket,
|
||||
object = %object.object_name,
|
||||
version_id = ?version_id,
|
||||
error = ?err,
|
||||
"failed to get object info during lifecycle noncurrent version cleanup; skipping delete replication scheduling"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
replication_candidates.push(candidate);
|
||||
}
|
||||
|
||||
let (mut deleted_objs, errors) = api
|
||||
.delete_objects(
|
||||
bucket,
|
||||
to_del.to_vec(),
|
||||
ObjectOptions {
|
||||
version_suspended,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
for (i, deleted_obj) in deleted_objs.iter_mut().enumerate() {
|
||||
if errors.get(i).and_then(|err| err.as_ref()).is_some() {
|
||||
continue;
|
||||
}
|
||||
let Some(replication_state) = replication_candidates.get(i).and_then(|c| c.clone()) else {
|
||||
continue;
|
||||
};
|
||||
deleted_obj.replication_state = Some(replication_state);
|
||||
schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||
delete_object: deleted_obj.clone(),
|
||||
bucket: bucket.to_string(),
|
||||
event_type: REPLICATE_INCOMING_DELETE.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
for (i, err) in errors.iter().enumerate() {
|
||||
if let Some(e) = err {
|
||||
let obj_name = to_del.get(i).map(|o| o.object_name.as_str()).unwrap_or("<unknown>");
|
||||
let vid = to_del
|
||||
.get(i)
|
||||
.and_then(|o| o.version_id)
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_default();
|
||||
warn!(bucket, object = obj_name, version_id = %vid, error = ?e, "failed to delete noncurrent version during lifecycle cleanup");
|
||||
}
|
||||
}
|
||||
if remaining.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::lifecycle_version_delete_replication_state;
|
||||
|
||||
#[test]
|
||||
fn lifecycle_version_delete_replication_state_tracks_pending_purge_targets() {
|
||||
let state = lifecycle_version_delete_replication_state(
|
||||
"arn:aws:s3:::target=true;false;arn:aws:s3:::target;".to_string(),
|
||||
Some("arn:aws:s3:::target=PENDING;".to_string()),
|
||||
);
|
||||
|
||||
assert_eq!(state.version_purge_status_internal.as_deref(), Some("arn:aws:s3:::target=PENDING;"));
|
||||
assert!(state.purge_targets.contains_key("arn:aws:s3:::target"));
|
||||
assert_eq!(state.replicate_decision_str, "arn:aws:s3:::target=true;false;arn:aws:s3:::target;");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,9 +251,10 @@ impl TransitionClient {
|
||||
};
|
||||
|
||||
{
|
||||
let mut md5_hasher = client.md5_hasher.lock().unwrap();
|
||||
if md5_hasher.is_none() {
|
||||
*md5_hasher = Some(HashAlgorithm::Md5);
|
||||
if let Ok(mut md5_hasher) = client.md5_hasher.lock() {
|
||||
if md5_hasher.is_none() {
|
||||
*md5_hasher = Some(HashAlgorithm::Md5);
|
||||
}
|
||||
}
|
||||
}
|
||||
if client.sha256_hasher.is_none() {
|
||||
@@ -275,25 +276,30 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
fn trace_errors_only_off(&self) {
|
||||
let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
|
||||
*trace_errors_only = false;
|
||||
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
||||
*trace_errors_only = false;
|
||||
}
|
||||
}
|
||||
|
||||
fn trace_off(&self) {
|
||||
let mut is_trace_enabled = self.is_trace_enabled.lock().unwrap();
|
||||
*is_trace_enabled = false;
|
||||
let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
|
||||
*trace_errors_only = false;
|
||||
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
|
||||
*is_trace_enabled = false;
|
||||
}
|
||||
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
||||
*trace_errors_only = false;
|
||||
}
|
||||
}
|
||||
|
||||
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
|
||||
let mut endpoint = self.s3_accelerate_endpoint.lock().unwrap();
|
||||
*endpoint = accelerate_endpoint.to_string();
|
||||
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
|
||||
*endpoint = accelerate_endpoint.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
fn set_s3_enable_dual_stack(&self, enabled: bool) {
|
||||
let mut dual_stack = self.s3_dual_stack_enabled.lock().unwrap();
|
||||
*dual_stack = enabled;
|
||||
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
|
||||
*dual_stack = enabled;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn hash_materials(
|
||||
@@ -352,7 +358,6 @@ impl TransitionClient {
|
||||
let resp;
|
||||
let http_client = self.http_client.clone();
|
||||
{
|
||||
//let mut http_client = http_client.lock().unwrap();
|
||||
req_method = req.method().clone();
|
||||
req_uri = req.uri().clone();
|
||||
req_headers = req.headers().clone();
|
||||
@@ -368,7 +373,10 @@ impl TransitionClient {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
|
||||
let resp = resp.unwrap();
|
||||
let resp = match resp {
|
||||
Ok(r) => r,
|
||||
Err(_) => return Err(std::io::Error::other("Unexpected error in response")),
|
||||
};
|
||||
debug!("http_resp: {:?}", resp);
|
||||
|
||||
//let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
|
||||
@@ -455,11 +463,13 @@ impl TransitionClient {
|
||||
return Err(std::io::Error::other(err_response));
|
||||
}
|
||||
if metadata.bucket_name != "" {
|
||||
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
|
||||
let location = bucket_loc_cache.get(&metadata.bucket_name);
|
||||
if location.is_some() && location.unwrap() != err_response.region {
|
||||
bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
|
||||
//continue;
|
||||
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
|
||||
if let Some(location) = bucket_loc_cache.get(&metadata.bucket_name) {
|
||||
if location != err_response.region {
|
||||
bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
|
||||
//continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if err_response.region != metadata.bucket_location {
|
||||
metadata.bucket_location = err_response.region.clone();
|
||||
@@ -518,8 +528,11 @@ impl TransitionClient {
|
||||
|
||||
let value;
|
||||
{
|
||||
let mut creds_provider = self.creds_provider.lock().unwrap();
|
||||
value = creds_provider.get_with_context(Some(self.cred_context()))?;
|
||||
if let Ok(mut creds_provider) = self.creds_provider.lock() {
|
||||
value = creds_provider.get_with_context(Some(self.cred_context()))?;
|
||||
} else {
|
||||
return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
|
||||
}
|
||||
}
|
||||
|
||||
let mut signer_type = value.signer_type.clone();
|
||||
@@ -548,8 +561,10 @@ impl TransitionClient {
|
||||
)));
|
||||
}
|
||||
let headers = req.headers_mut();
|
||||
for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() {
|
||||
headers.insert(k, v.clone());
|
||||
if let Some(extra_headers) = metadata.extra_pre_sign_header.as_ref() {
|
||||
for (k, v) in extra_headers {
|
||||
headers.insert(k, v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
if signer_type == SignatureType::SignatureV2 {
|
||||
@@ -571,18 +586,22 @@ impl TransitionClient {
|
||||
self.set_user_agent(&mut req);
|
||||
|
||||
for (k, v) in metadata.custom_header.clone() {
|
||||
req.headers_mut().insert(k.expect("err"), v);
|
||||
if let Some(key) = k {
|
||||
req.headers_mut().insert(key, v);
|
||||
}
|
||||
}
|
||||
|
||||
//req.content_length = metadata.content_length;
|
||||
if metadata.content_length <= -1 {
|
||||
let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
|
||||
req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
|
||||
if let Ok(chunked_value) = HeaderValue::from_str(&vec!["chunked"].join(",")) {
|
||||
req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
|
||||
}
|
||||
}
|
||||
|
||||
if metadata.content_md5_base64.len() > 0 {
|
||||
let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).expect("err");
|
||||
req.headers_mut().insert("Content-Md5", md5_value);
|
||||
if let Ok(md5_value) = HeaderValue::from_str(&metadata.content_md5_base64) {
|
||||
req.headers_mut().insert("Content-Md5", md5_value);
|
||||
}
|
||||
}
|
||||
|
||||
if signer_type == SignatureType::SignatureAnonymous {
|
||||
@@ -607,8 +626,13 @@ impl TransitionClient {
|
||||
} else if metadata.trailer.len() > 0 {
|
||||
sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
|
||||
}
|
||||
req.headers_mut()
|
||||
.insert("X-Amz-Content-Sha256".parse::<HeaderName>().unwrap(), sha_header.parse().expect("err"));
|
||||
let header_name = "X-Amz-Content-Sha256"
|
||||
.parse::<HeaderName>()
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
|
||||
let header_value = sha_header
|
||||
.parse()
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
|
||||
req.headers_mut().insert(header_name, header_value);
|
||||
|
||||
req = rustfs_signer::sign_v4_trailer(
|
||||
req,
|
||||
@@ -636,7 +660,7 @@ impl TransitionClient {
|
||||
|
||||
pub fn set_user_agent(&self, req: &mut Request<s3s::Body>) {
|
||||
let headers = req.headers_mut();
|
||||
headers.insert("User-Agent", C_USER_AGENT.parse().expect("err"));
|
||||
headers.insert("User-Agent", HeaderValue::from_static(C_USER_AGENT));
|
||||
}
|
||||
|
||||
fn make_target_url(
|
||||
@@ -648,7 +672,10 @@ impl TransitionClient {
|
||||
query_values: &HashMap<String, String>,
|
||||
) -> Result<Url, std::io::Error> {
|
||||
let scheme = self.endpoint_url.scheme();
|
||||
let host = self.endpoint_url.host().unwrap();
|
||||
let host = self
|
||||
.endpoint_url
|
||||
.host()
|
||||
.ok_or_else(|| std::io::Error::other("Endpoint URL has no host"))?;
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let port = self.endpoint_url.port().unwrap_or(default_port);
|
||||
|
||||
@@ -1155,9 +1182,10 @@ pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Re
|
||||
for (name, value) in h.iter() {
|
||||
let header_name = name.as_str().to_lowercase();
|
||||
if header_name.starts_with("x-amz-meta-") {
|
||||
let key = header_name.strip_prefix("x-amz-meta-").unwrap().to_string();
|
||||
if let Ok(value_str) = value.to_str() {
|
||||
meta.insert(key, value_str.to_string());
|
||||
if let Some(key) = header_name.strip_prefix("x-amz-meta-") {
|
||||
if let Ok(value_str) = value.to_str() {
|
||||
meta.insert(key.to_string(), value_str.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,9 +16,13 @@ use crate::config::{KV, KVS};
|
||||
use rustfs_config::{
|
||||
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
|
||||
MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
|
||||
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
|
||||
WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT,
|
||||
WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
|
||||
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
|
||||
NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
|
||||
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
|
||||
PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
|
||||
PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
|
||||
WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
|
||||
WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
@@ -192,3 +196,144 @@ pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_ADDRESS.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_SUBJECT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TOKEN.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_CREDENTIALS_FILE.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_REQUIRED.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_BROKER.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TOPIC.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_AUTH_TOKEN.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
|
||||
value: EnableState::On.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
+175
-129
@@ -16,10 +16,16 @@ use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, audit, notify, oidc, stor
|
||||
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::global::is_first_cluster_node_local;
|
||||
use crate::store_api::{ChunkNativePutData, ObjectInfo, ObjectOptions, StorageAPI};
|
||||
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
|
||||
use http::HeaderMap;
|
||||
use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::audit::{
|
||||
AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
|
||||
AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
|
||||
};
|
||||
use rustfs_config::notify::{
|
||||
NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS,
|
||||
NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
|
||||
};
|
||||
use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
|
||||
use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
@@ -34,6 +40,8 @@ const CONFIG_FILE: &str = "config.json";
|
||||
|
||||
pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
|
||||
|
||||
pub const COMMA_SEPARATED_LISTS: &[&str] = &[rustfs_config::oidc::OIDC_SCOPES, rustfs_config::oidc::OIDC_OTHER_AUDIENCES];
|
||||
|
||||
static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}"));
|
||||
|
||||
static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
|
||||
@@ -42,6 +50,72 @@ static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
|
||||
h
|
||||
});
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct TargetConfigDescriptor {
|
||||
external_key: &'static str,
|
||||
subsystem_key: &'static str,
|
||||
default_kvs: &'static LazyLock<KVS>,
|
||||
valid_keys: &'static [&'static str],
|
||||
}
|
||||
|
||||
fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
|
||||
[
|
||||
TargetConfigDescriptor {
|
||||
external_key: "webhook",
|
||||
subsystem_key: NOTIFY_WEBHOOK_SUB_SYS,
|
||||
default_kvs: ¬ify::DEFAULT_NOTIFY_WEBHOOK_KVS,
|
||||
valid_keys: NOTIFY_WEBHOOK_KEYS,
|
||||
},
|
||||
TargetConfigDescriptor {
|
||||
external_key: "mqtt",
|
||||
subsystem_key: NOTIFY_MQTT_SUB_SYS,
|
||||
default_kvs: ¬ify::DEFAULT_NOTIFY_MQTT_KVS,
|
||||
valid_keys: NOTIFY_MQTT_KEYS,
|
||||
},
|
||||
TargetConfigDescriptor {
|
||||
external_key: "nats",
|
||||
subsystem_key: NOTIFY_NATS_SUB_SYS,
|
||||
default_kvs: ¬ify::DEFAULT_NOTIFY_NATS_KVS,
|
||||
valid_keys: NOTIFY_NATS_KEYS,
|
||||
},
|
||||
TargetConfigDescriptor {
|
||||
external_key: "pulsar",
|
||||
subsystem_key: NOTIFY_PULSAR_SUB_SYS,
|
||||
default_kvs: ¬ify::DEFAULT_NOTIFY_PULSAR_KVS,
|
||||
valid_keys: NOTIFY_PULSAR_KEYS,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn audit_target_descriptors() -> [TargetConfigDescriptor; 4] {
|
||||
[
|
||||
TargetConfigDescriptor {
|
||||
external_key: "webhook",
|
||||
subsystem_key: AUDIT_WEBHOOK_SUB_SYS,
|
||||
default_kvs: &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
|
||||
valid_keys: AUDIT_WEBHOOK_KEYS,
|
||||
},
|
||||
TargetConfigDescriptor {
|
||||
external_key: "mqtt",
|
||||
subsystem_key: AUDIT_MQTT_SUB_SYS,
|
||||
default_kvs: &audit::DEFAULT_AUDIT_MQTT_KVS,
|
||||
valid_keys: AUDIT_MQTT_KEYS,
|
||||
},
|
||||
TargetConfigDescriptor {
|
||||
external_key: "nats",
|
||||
subsystem_key: AUDIT_NATS_SUB_SYS,
|
||||
default_kvs: &audit::DEFAULT_AUDIT_NATS_KVS,
|
||||
valid_keys: AUDIT_NATS_KEYS,
|
||||
},
|
||||
TargetConfigDescriptor {
|
||||
external_key: "pulsar",
|
||||
subsystem_key: AUDIT_PULSAR_SUB_SYS,
|
||||
default_kvs: &audit::DEFAULT_AUDIT_PULSAR_KVS,
|
||||
valid_keys: AUDIT_PULSAR_KEYS,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[instrument(skip(api))]
|
||||
pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
|
||||
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
|
||||
@@ -128,7 +202,7 @@ pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()>
|
||||
}
|
||||
|
||||
pub async fn save_config_with_opts<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
|
||||
let mut put_data = ChunkNativePutData::from_vec(data);
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
|
||||
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
|
||||
return Err(err);
|
||||
@@ -192,15 +266,15 @@ fn parse_oidc_scalar_value(key: &str, value: &Value) -> Option<String> {
|
||||
}),
|
||||
Value::Bool(v) => Some(v.to_string()),
|
||||
Value::Number(v) => Some(v.to_string()),
|
||||
Value::Array(values) if key == rustfs_config::oidc::OIDC_SCOPES => {
|
||||
let scopes = values
|
||||
Value::Array(values) if COMMA_SEPARATED_LISTS.contains(&key) => {
|
||||
let values_str = values
|
||||
.iter()
|
||||
.filter_map(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|scope| !scope.is_empty())
|
||||
.filter(|val| !val.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
Some(scopes)
|
||||
Some(values_str)
|
||||
}
|
||||
Value::Null => None,
|
||||
_ => None,
|
||||
@@ -363,29 +437,31 @@ fn apply_external_notify_section(
|
||||
applied
|
||||
}
|
||||
|
||||
fn apply_external_target_descriptors(
|
||||
cfg: &mut Config,
|
||||
section_obj: &Map<String, Value>,
|
||||
descriptors: &[TargetConfigDescriptor],
|
||||
) -> bool {
|
||||
let mut applied = false;
|
||||
for descriptor in descriptors {
|
||||
applied |= apply_external_notify_section(
|
||||
cfg,
|
||||
section_obj,
|
||||
descriptor.external_key,
|
||||
descriptor.subsystem_key,
|
||||
descriptor.default_kvs,
|
||||
descriptor.valid_keys,
|
||||
);
|
||||
}
|
||||
applied
|
||||
}
|
||||
|
||||
fn apply_external_notify_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
|
||||
let Some(Value::Object(notify_obj)) = root.get("notify") else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let mut applied = false;
|
||||
applied |= apply_external_notify_section(
|
||||
cfg,
|
||||
notify_obj,
|
||||
"webhook",
|
||||
NOTIFY_WEBHOOK_SUB_SYS,
|
||||
¬ify::DEFAULT_NOTIFY_WEBHOOK_KVS,
|
||||
NOTIFY_WEBHOOK_KEYS,
|
||||
);
|
||||
applied |= apply_external_notify_section(
|
||||
cfg,
|
||||
notify_obj,
|
||||
"mqtt",
|
||||
NOTIFY_MQTT_SUB_SYS,
|
||||
¬ify::DEFAULT_NOTIFY_MQTT_KVS,
|
||||
NOTIFY_MQTT_KEYS,
|
||||
);
|
||||
applied
|
||||
apply_external_target_descriptors(cfg, notify_obj, ¬ify_target_descriptors())
|
||||
}
|
||||
|
||||
fn apply_external_audit_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
|
||||
@@ -394,24 +470,7 @@ fn apply_external_audit_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
|
||||
return false;
|
||||
};
|
||||
|
||||
let mut applied = false;
|
||||
applied |= apply_external_notify_section(
|
||||
cfg,
|
||||
audit_obj,
|
||||
"webhook",
|
||||
AUDIT_WEBHOOK_SUB_SYS,
|
||||
&audit::DEFAULT_AUDIT_WEBHOOK_KVS,
|
||||
AUDIT_WEBHOOK_KEYS,
|
||||
);
|
||||
applied |= apply_external_notify_section(
|
||||
cfg,
|
||||
audit_obj,
|
||||
"mqtt",
|
||||
AUDIT_MQTT_SUB_SYS,
|
||||
&audit::DEFAULT_AUDIT_MQTT_KVS,
|
||||
AUDIT_MQTT_KEYS,
|
||||
);
|
||||
applied
|
||||
apply_external_target_descriptors(cfg, audit_obj, &audit_target_descriptors())
|
||||
}
|
||||
|
||||
fn apply_external_storage_class_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
|
||||
@@ -513,15 +572,15 @@ fn build_oidc_provider_object(kvs: &KVS) -> Map<String, Value> {
|
||||
continue;
|
||||
}
|
||||
|
||||
if kv.key == rustfs_config::oidc::OIDC_SCOPES {
|
||||
let scopes = kv
|
||||
if COMMA_SEPARATED_LISTS.contains(&kv.key.as_str()) {
|
||||
let values = kv
|
||||
.value
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|scope| !scope.is_empty())
|
||||
.map(|scope| Value::String(scope.to_string()))
|
||||
.filter(|val| !val.is_empty())
|
||||
.map(|val| Value::String(val.to_string()))
|
||||
.collect::<Vec<_>>();
|
||||
provider.insert(kv.key.clone(), Value::Array(scopes));
|
||||
provider.insert(kv.key.clone(), Value::Array(values));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -537,7 +596,7 @@ fn build_oidc_object(cfg: &Config) -> Map<String, Value> {
|
||||
};
|
||||
|
||||
let mut providers = subsystem.iter().collect::<Vec<_>>();
|
||||
providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
|
||||
providers.sort_by_key(|(lhs, _)| *lhs);
|
||||
|
||||
let mut oidc_obj = Map::new();
|
||||
for (instance_key, kvs) in providers {
|
||||
@@ -572,7 +631,7 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
|
||||
};
|
||||
|
||||
let mut providers = subsystem.iter().collect::<Vec<_>>();
|
||||
providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
|
||||
providers.sort_by_key(|(lhs, _)| *lhs);
|
||||
|
||||
let mut oidc_obj = Map::new();
|
||||
for (instance_key, kvs) in providers {
|
||||
@@ -605,7 +664,15 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
|
||||
}
|
||||
|
||||
fn is_notify_bool_key(key: &str) -> bool {
|
||||
key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
|
||||
matches!(
|
||||
key,
|
||||
ENABLE_KEY
|
||||
| rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
|
||||
| rustfs_config::MQTT_TLS_TRUST_LEAF_AS_CA
|
||||
| rustfs_config::NATS_TLS_REQUIRED
|
||||
| rustfs_config::PULSAR_TLS_ALLOW_INSECURE
|
||||
| rustfs_config::PULSAR_TLS_HOSTNAME_VERIFICATION
|
||||
)
|
||||
}
|
||||
|
||||
fn encode_notify_scalar_value(key: &str, value: &str) -> Value {
|
||||
@@ -691,7 +758,7 @@ fn build_notify_subsystem_object(
|
||||
.iter()
|
||||
.filter(|(instance_key, _)| instance_key.as_str() != DEFAULT_DELIMITER)
|
||||
.collect::<Vec<_>>();
|
||||
instances.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
|
||||
instances.sort_by_key(|(lhs, _)| *lhs);
|
||||
|
||||
for (instance_key, kvs) in instances {
|
||||
let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
|
||||
@@ -703,38 +770,43 @@ fn build_notify_subsystem_object(
|
||||
subsystem_obj
|
||||
}
|
||||
|
||||
fn build_target_object(cfg: &Config, descriptors: &[TargetConfigDescriptor]) -> Map<String, Value> {
|
||||
let mut target_obj = Map::new();
|
||||
for descriptor in descriptors {
|
||||
let subsystem_obj =
|
||||
build_notify_subsystem_object(cfg, descriptor.subsystem_key, descriptor.default_kvs, descriptor.valid_keys);
|
||||
if !subsystem_obj.is_empty() {
|
||||
target_obj.insert(descriptor.external_key.to_string(), Value::Object(subsystem_obj));
|
||||
}
|
||||
}
|
||||
target_obj
|
||||
}
|
||||
|
||||
fn build_notify_object(cfg: &Config) -> Map<String, Value> {
|
||||
let mut notify_obj = Map::new();
|
||||
|
||||
let webhook_obj =
|
||||
build_notify_subsystem_object(cfg, NOTIFY_WEBHOOK_SUB_SYS, ¬ify::DEFAULT_NOTIFY_WEBHOOK_KVS, NOTIFY_WEBHOOK_KEYS);
|
||||
if !webhook_obj.is_empty() {
|
||||
notify_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
|
||||
}
|
||||
|
||||
let mqtt_obj = build_notify_subsystem_object(cfg, NOTIFY_MQTT_SUB_SYS, ¬ify::DEFAULT_NOTIFY_MQTT_KVS, NOTIFY_MQTT_KEYS);
|
||||
if !mqtt_obj.is_empty() {
|
||||
notify_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
|
||||
}
|
||||
|
||||
notify_obj
|
||||
build_target_object(cfg, ¬ify_target_descriptors())
|
||||
}
|
||||
|
||||
fn build_audit_object(cfg: &Config) -> Map<String, Value> {
|
||||
let mut audit_obj = Map::new();
|
||||
build_target_object(cfg, &audit_target_descriptors())
|
||||
}
|
||||
|
||||
let webhook_obj =
|
||||
build_notify_subsystem_object(cfg, AUDIT_WEBHOOK_SUB_SYS, &audit::DEFAULT_AUDIT_WEBHOOK_KVS, AUDIT_WEBHOOK_KEYS);
|
||||
if !webhook_obj.is_empty() {
|
||||
audit_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
|
||||
fn sync_rendered_target_object(
|
||||
target_obj: &mut Map<String, Value>,
|
||||
rendered_target: &Map<String, Value>,
|
||||
descriptors: &[TargetConfigDescriptor],
|
||||
) {
|
||||
for descriptor in descriptors {
|
||||
match rendered_target.get(descriptor.external_key) {
|
||||
Some(Value::Object(v)) => {
|
||||
target_obj.insert(descriptor.external_key.to_string(), Value::Object(v.clone()));
|
||||
target_obj.remove(descriptor.subsystem_key);
|
||||
}
|
||||
_ => {
|
||||
target_obj.remove(descriptor.external_key);
|
||||
target_obj.remove(descriptor.subsystem_key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mqtt_obj = build_notify_subsystem_object(cfg, AUDIT_MQTT_SUB_SYS, &audit::DEFAULT_AUDIT_MQTT_KVS, AUDIT_MQTT_KEYS);
|
||||
if !mqtt_obj.is_empty() {
|
||||
audit_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
|
||||
}
|
||||
|
||||
audit_obj
|
||||
}
|
||||
|
||||
fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8>> {
|
||||
@@ -771,67 +843,31 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
|
||||
_ => Map::new(),
|
||||
};
|
||||
let rendered_notify = build_notify_object(cfg);
|
||||
match rendered_notify.get("webhook") {
|
||||
Some(Value::Object(v)) => {
|
||||
notify_obj.insert("webhook".to_string(), Value::Object(v.clone()));
|
||||
notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
|
||||
}
|
||||
_ => {
|
||||
notify_obj.remove("webhook");
|
||||
notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
|
||||
}
|
||||
}
|
||||
match rendered_notify.get("mqtt") {
|
||||
Some(Value::Object(v)) => {
|
||||
notify_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
|
||||
notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
|
||||
}
|
||||
_ => {
|
||||
notify_obj.remove("mqtt");
|
||||
notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
|
||||
}
|
||||
}
|
||||
sync_rendered_target_object(&mut notify_obj, &rendered_notify, ¬ify_target_descriptors());
|
||||
if notify_obj.is_empty() {
|
||||
root.remove("notify");
|
||||
} else {
|
||||
root.insert("notify".to_string(), Value::Object(notify_obj));
|
||||
}
|
||||
root.remove(NOTIFY_WEBHOOK_SUB_SYS);
|
||||
root.remove(NOTIFY_MQTT_SUB_SYS);
|
||||
for descriptor in notify_target_descriptors() {
|
||||
root.remove(descriptor.subsystem_key);
|
||||
}
|
||||
|
||||
let mut logger_obj = match root.remove("logger") {
|
||||
Some(Value::Object(v)) => v,
|
||||
_ => Map::new(),
|
||||
};
|
||||
let rendered_audit = build_audit_object(cfg);
|
||||
match rendered_audit.get("webhook") {
|
||||
Some(Value::Object(v)) => {
|
||||
logger_obj.insert("webhook".to_string(), Value::Object(v.clone()));
|
||||
logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
|
||||
}
|
||||
_ => {
|
||||
logger_obj.remove("webhook");
|
||||
logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
|
||||
}
|
||||
}
|
||||
match rendered_audit.get("mqtt") {
|
||||
Some(Value::Object(v)) => {
|
||||
logger_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
|
||||
logger_obj.remove(AUDIT_MQTT_SUB_SYS);
|
||||
}
|
||||
_ => {
|
||||
logger_obj.remove("mqtt");
|
||||
logger_obj.remove(AUDIT_MQTT_SUB_SYS);
|
||||
}
|
||||
}
|
||||
sync_rendered_target_object(&mut logger_obj, &rendered_audit, &audit_target_descriptors());
|
||||
if logger_obj.is_empty() {
|
||||
root.remove("logger");
|
||||
} else {
|
||||
root.insert("logger".to_string(), Value::Object(logger_obj));
|
||||
}
|
||||
root.remove("audit");
|
||||
root.remove(AUDIT_WEBHOOK_SUB_SYS);
|
||||
root.remove(AUDIT_MQTT_SUB_SYS);
|
||||
for descriptor in audit_target_descriptors() {
|
||||
root.remove(descriptor.subsystem_key);
|
||||
}
|
||||
|
||||
Ok(serde_json::to_vec(&Value::Object(root))?)
|
||||
}
|
||||
@@ -1076,10 +1112,10 @@ mod tests {
|
||||
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::store_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, DeleteBucketOptions, DeletedObject,
|
||||
GetObjectReader, HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info,
|
||||
ListOperations, MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo,
|
||||
ObjectOperations, ObjectOptions, ObjectToDelete, PartInfo, StorageAPI, WalkOptions,
|
||||
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader,
|
||||
HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations,
|
||||
MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations,
|
||||
ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
|
||||
@@ -1302,7 +1338,7 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut ChunkNativePutData,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
panic!("unused in test")
|
||||
@@ -1489,7 +1525,7 @@ mod tests {
|
||||
_object: &str,
|
||||
_upload_id: &str,
|
||||
_part_id: usize,
|
||||
_data: &mut ChunkNativePutData,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
panic!("unused in test")
|
||||
@@ -1655,6 +1691,7 @@ mod tests {
|
||||
"client_id":"console",
|
||||
"client_secret":"secret-value",
|
||||
"scopes":["openid","profile","email"],
|
||||
"other_audiences":["aud1", "aud2"],
|
||||
"redirect_uri_dynamic":true,
|
||||
"display_name":"Default Provider"
|
||||
},
|
||||
@@ -1679,6 +1716,7 @@ mod tests {
|
||||
);
|
||||
assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_CLIENT_ID), "console");
|
||||
assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_SCOPES), "openid,profile,email");
|
||||
assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_OTHER_AUDIENCES), "aud1,aud2");
|
||||
assert_eq!(default_kvs.get(ENABLE_KEY), EnableState::On.to_string());
|
||||
|
||||
let smoke_kvs = cfg
|
||||
@@ -1863,6 +1901,7 @@ mod tests {
|
||||
);
|
||||
default_provider.insert(rustfs_config::oidc::OIDC_CLIENT_ID.to_string(), "console".to_string());
|
||||
default_provider.insert(rustfs_config::oidc::OIDC_SCOPES.to_string(), "openid,profile,email".to_string());
|
||||
default_provider.insert(rustfs_config::oidc::OIDC_OTHER_AUDIENCES.to_string(), "aud1,aud2".to_string());
|
||||
oidc_section.insert(DEFAULT_DELIMITER.to_string(), default_provider);
|
||||
cfg.0.insert(IDENTITY_OPENID_SUB_SYS.to_string(), oidc_section);
|
||||
|
||||
@@ -1890,6 +1929,13 @@ mod tests {
|
||||
.map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
|
||||
Some(vec!["openid", "profile", "email"])
|
||||
);
|
||||
assert_eq!(
|
||||
default_provider
|
||||
.get(rustfs_config::oidc::OIDC_OTHER_AUDIENCES)
|
||||
.and_then(Value::as_array)
|
||||
.map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
|
||||
Some(vec!["aud1", "aud2"])
|
||||
);
|
||||
assert_eq!(default_provider.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
|
||||
}
|
||||
|
||||
|
||||
@@ -25,8 +25,8 @@ use crate::store::ECStore;
|
||||
use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
|
||||
use rustfs_config::COMMENT_KEY;
|
||||
use rustfs_config::DEFAULT_DELIMITER;
|
||||
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
@@ -144,7 +144,7 @@ impl KVS {
|
||||
pub fn insert(&mut self, key: String, value: String) {
|
||||
for kv in self.0.iter_mut() {
|
||||
if kv.key == key {
|
||||
kv.value = value.clone();
|
||||
kv.value = value;
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -241,6 +241,10 @@ pub fn init() {
|
||||
kvs.insert(AUDIT_WEBHOOK_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone());
|
||||
kvs.insert(NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_MQTT_KVS.clone());
|
||||
kvs.insert(AUDIT_MQTT_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MQTT_KVS.clone());
|
||||
kvs.insert(NOTIFY_NATS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_NATS_KVS.clone());
|
||||
kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone());
|
||||
kvs.insert(NOTIFY_PULSAR_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_PULSAR_KVS.clone());
|
||||
kvs.insert(AUDIT_PULSAR_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_PULSAR_KVS.clone());
|
||||
kvs.insert(IDENTITY_OPENID_SUB_SYS.to_owned(), oidc::DEFAULT_IDENTITY_OPENID_KVS.clone());
|
||||
|
||||
// Register all default configurations
|
||||
|
||||
@@ -16,9 +16,12 @@ use crate::config::{KV, KVS};
|
||||
use rustfs_config::{
|
||||
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
|
||||
MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
|
||||
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
|
||||
WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
|
||||
WEBHOOK_SKIP_TLS_VERIFY,
|
||||
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
|
||||
NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
|
||||
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
|
||||
PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
|
||||
PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
|
||||
WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
@@ -171,3 +174,143 @@ pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_ADDRESS.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_SUBJECT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TOKEN.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_CREDENTIALS_FILE.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_REQUIRED.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_BROKER.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TOPIC.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_AUTH_TOKEN.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
|
||||
value: EnableState::On.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
@@ -17,9 +17,9 @@ use rustfs_config::{
|
||||
ENABLE_KEY, EnableState,
|
||||
oidc::{
|
||||
OIDC_CLAIM_NAME, OIDC_CLAIM_PREFIX, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_CONFIG_URL, OIDC_DEFAULT_CLAIM_NAME,
|
||||
OIDC_DEFAULT_EMAIL_CLAIM, OIDC_DEFAULT_GROUPS_CLAIM, OIDC_DEFAULT_SCOPES, OIDC_DEFAULT_USERNAME_CLAIM, OIDC_DISPLAY_NAME,
|
||||
OIDC_EMAIL_CLAIM, OIDC_GROUPS_CLAIM, OIDC_REDIRECT_URI, OIDC_REDIRECT_URI_DYNAMIC, OIDC_ROLE_POLICY, OIDC_SCOPES,
|
||||
OIDC_USERNAME_CLAIM,
|
||||
OIDC_DEFAULT_EMAIL_CLAIM, OIDC_DEFAULT_GROUPS_CLAIM, OIDC_DEFAULT_ROLES_CLAIM, OIDC_DEFAULT_SCOPES,
|
||||
OIDC_DEFAULT_USERNAME_CLAIM, OIDC_DISPLAY_NAME, OIDC_EMAIL_CLAIM, OIDC_GROUPS_CLAIM, OIDC_OTHER_AUDIENCES,
|
||||
OIDC_REDIRECT_URI, OIDC_REDIRECT_URI_DYNAMIC, OIDC_ROLE_POLICY, OIDC_ROLES_CLAIM, OIDC_SCOPES, OIDC_USERNAME_CLAIM,
|
||||
},
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
@@ -52,6 +52,11 @@ pub static DEFAULT_IDENTITY_OPENID_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
value: OIDC_DEFAULT_SCOPES.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: OIDC_OTHER_AUDIENCES.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: OIDC_REDIRECT_URI.to_owned(),
|
||||
value: "".to_owned(),
|
||||
@@ -87,6 +92,11 @@ pub static DEFAULT_IDENTITY_OPENID_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
value: OIDC_DEFAULT_GROUPS_CLAIM.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: OIDC_ROLES_CLAIM.to_owned(),
|
||||
value: OIDC_DEFAULT_ROLES_CLAIM.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: OIDC_EMAIL_CLAIM.to_owned(),
|
||||
value: OIDC_DEFAULT_EMAIL_CLAIM.to_owned(),
|
||||
|
||||
@@ -150,7 +150,18 @@ impl Config {
|
||||
return false;
|
||||
}
|
||||
|
||||
shard_size as usize <= self.inline_shard_limit_bytes(versioned)
|
||||
let shard_size = shard_size as usize;
|
||||
|
||||
let mut inline_block = DEFAULT_INLINE_BLOCK;
|
||||
if self.initialized {
|
||||
inline_block = self.inline_block;
|
||||
}
|
||||
|
||||
if versioned {
|
||||
shard_size <= inline_block / 8
|
||||
} else {
|
||||
shard_size <= inline_block
|
||||
}
|
||||
}
|
||||
|
||||
pub fn inline_block(&self) -> usize {
|
||||
@@ -161,15 +172,6 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn inline_shard_limit_bytes(&self, versioned: bool) -> usize {
|
||||
let inline_block = self.inline_block();
|
||||
if versioned { inline_block / 8 } else { inline_block }
|
||||
}
|
||||
|
||||
pub fn inline_object_limit_bytes(&self, data_shards: usize, versioned: bool) -> usize {
|
||||
self.inline_shard_limit_bytes(versioned).saturating_mul(data_shards.max(1))
|
||||
}
|
||||
|
||||
pub fn capacity_optimized(&self) -> bool {
|
||||
if !self.initialized {
|
||||
false
|
||||
@@ -334,32 +336,3 @@ pub fn validate_parity_inner(ss_parity: usize, rrs_parity: usize, set_drive_coun
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn inline_object_limit_matches_default_non_versioned_budget() {
|
||||
let cfg = Config {
|
||||
initialized: true,
|
||||
inline_block: DEFAULT_INLINE_BLOCK,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(cfg.inline_shard_limit_bytes(false), DEFAULT_INLINE_BLOCK);
|
||||
assert_eq!(cfg.inline_object_limit_bytes(8, false), DEFAULT_INLINE_BLOCK * 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_object_limit_scales_down_for_versioned_objects() {
|
||||
let cfg = Config {
|
||||
initialized: true,
|
||||
inline_block: DEFAULT_INLINE_BLOCK,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(cfg.inline_shard_limit_bytes(true), DEFAULT_INLINE_BLOCK / 8);
|
||||
assert_eq!(cfg.inline_object_limit_bytes(8, true), DEFAULT_INLINE_BLOCK);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,11 +14,9 @@
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{
|
||||
ChunkNativePutData, CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions,
|
||||
};
|
||||
use crate::store_api::{CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use bytes::Bytes;
|
||||
use rustfs_rio::{BlockReadable, BoxReadBlockFuture, EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
|
||||
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
|
||||
use std::io::Cursor;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{
|
||||
@@ -56,11 +54,6 @@ impl<R: AsyncRead + Unpin + Send + Sync> TryGetIndex for IndexedDataMovementRead
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin + Send + Sync> BlockReadable for IndexedDataMovementReader<R> {
|
||||
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
|
||||
Box::pin(rustfs_utils::read_full(self, buf))
|
||||
}
|
||||
}
|
||||
pub fn decode_part_index(index: Option<&Bytes>) -> Option<Index> {
|
||||
let bytes = index?;
|
||||
let mut decoded = Index::new();
|
||||
@@ -71,7 +64,7 @@ pub fn decode_part_index(index: Option<&Bytes>) -> Option<Index> {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn put_data_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: Option<Index>) -> Result<ChunkNativePutData> {
|
||||
pub fn put_obj_reader_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: Option<Index>) -> Result<PutObjReader> {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
let sha256hex = if !chunk.is_empty() {
|
||||
@@ -81,8 +74,8 @@ pub fn put_data_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: O
|
||||
};
|
||||
|
||||
let reader = IndexedDataMovementReader::new(Cursor::new(chunk), index);
|
||||
let hash_reader = HashReader::from_reader(reader, size, actual_size, None, sha256hex, false)?;
|
||||
Ok(ChunkNativePutData::new(hash_reader))
|
||||
let hash_reader = HashReader::from_stream(reader, size, actual_size, None, sha256hex, false)?;
|
||||
Ok(PutObjReader::new(hash_reader))
|
||||
}
|
||||
|
||||
pub fn new_multipart_abort_flag() -> Arc<AtomicBool> {
|
||||
@@ -179,7 +172,7 @@ pub(crate) async fn migrate_object(
|
||||
let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?;
|
||||
let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size };
|
||||
let index = decode_part_index(part.index.as_ref());
|
||||
let mut data = put_data_from_chunk(chunk, part_size, part_actual_size, index)?;
|
||||
let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?;
|
||||
|
||||
let pi = match store
|
||||
.put_object_part(
|
||||
@@ -261,8 +254,8 @@ pub(crate) async fn migrate_object(
|
||||
.first()
|
||||
.and_then(|part| decode_part_index(part.index.as_ref()));
|
||||
let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index);
|
||||
let hrd = HashReader::from_reader(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?;
|
||||
let mut data = ChunkNativePutData::new(hrd);
|
||||
let hrd = HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?;
|
||||
let mut data = PutObjReader::new(hrd);
|
||||
|
||||
if let Err(err) = store
|
||||
.put_object(
|
||||
|
||||
@@ -16,12 +16,17 @@ use crate::disk::{
|
||||
CheckPartsResp, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint, Error,
|
||||
FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result, UpdateMetadataOpts, VolumeInfo,
|
||||
WalkDirOptions,
|
||||
health_state::{
|
||||
RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval,
|
||||
get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration,
|
||||
record_drive_recovery_class, record_drive_runtime_state, record_drive_state_transition,
|
||||
},
|
||||
local::{LocalDisk, ScanGuard},
|
||||
};
|
||||
use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
|
||||
use bytes::Bytes;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_io_core::BoxChunkStream;
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
@@ -41,7 +46,6 @@ const DISK_HEALTH_FAULTY: u32 = 1;
|
||||
|
||||
pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
|
||||
pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
|
||||
pub const ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
|
||||
pub const CHECK_EVERY: Duration = Duration::from_secs(15);
|
||||
pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
|
||||
pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5);
|
||||
@@ -52,9 +56,52 @@ lazy_static::lazy_static! {
|
||||
}
|
||||
|
||||
pub fn get_max_timeout_duration() -> Duration {
|
||||
std::env::var(ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION)
|
||||
.map(|v| Duration::from_secs(v.parse::<u64>().unwrap_or(30)))
|
||||
.unwrap_or(Duration::from_secs(30))
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION,
|
||||
rustfs_config::DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
fn get_drive_timeout_duration(env_key: &str, default_secs: u64) -> Duration {
|
||||
Duration::from_secs(
|
||||
rustfs_utils::get_env_opt_u64_with_aliases(env_key, &[rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION])
|
||||
.unwrap_or(default_secs),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_drive_metadata_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_drive_disk_info_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_DISK_INFO_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_drive_list_dir_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_LIST_DIR_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_drive_walkdir_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_drive_walkdir_stall_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
|
||||
)
|
||||
}
|
||||
|
||||
/// DiskHealthTracker tracks the health status of a disk.
|
||||
@@ -69,6 +116,16 @@ pub struct DiskHealthTracker {
|
||||
pub status: AtomicU32,
|
||||
/// Atomic number of waiting operations
|
||||
pub waiting: AtomicU32,
|
||||
/// Runtime drive health state
|
||||
pub runtime_state: AtomicU32,
|
||||
/// Consecutive failures while transitioning away from online
|
||||
pub consecutive_failures: AtomicU32,
|
||||
/// Consecutive successes while returning online
|
||||
pub consecutive_successes: AtomicU32,
|
||||
/// When the drive first left the online state
|
||||
pub offline_since_unix_secs: AtomicI64,
|
||||
/// Last runtime state transition timestamp
|
||||
pub last_transition_unix_secs: AtomicI64,
|
||||
}
|
||||
|
||||
impl DiskHealthTracker {
|
||||
@@ -84,6 +141,11 @@ impl DiskHealthTracker {
|
||||
last_started: AtomicI64::new(now),
|
||||
status: AtomicU32::new(DISK_HEALTH_OK),
|
||||
waiting: AtomicU32::new(0),
|
||||
runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32),
|
||||
consecutive_failures: AtomicU32::new(0),
|
||||
consecutive_successes: AtomicU32::new(0),
|
||||
offline_since_unix_secs: AtomicI64::new(0),
|
||||
last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,12 +173,139 @@ impl DiskHealthTracker {
|
||||
self.status.store(DISK_HEALTH_OK, Ordering::Release);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
|
||||
self.runtime_state.store(state as u32, Ordering::Release);
|
||||
match state {
|
||||
RuntimeDriveHealthState::Offline => self.set_faulty(),
|
||||
RuntimeDriveHealthState::Online | RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Returning => {
|
||||
self.set_ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn swap_ok_to_faulty(&self) -> bool {
|
||||
self.status
|
||||
.compare_exchange(DISK_HEALTH_OK, DISK_HEALTH_FAULTY, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
pub fn runtime_state(&self) -> RuntimeDriveHealthState {
|
||||
RuntimeDriveHealthState::from_u32(self.runtime_state.load(Ordering::Acquire))
|
||||
}
|
||||
|
||||
pub fn offline_duration(&self) -> Option<Duration> {
|
||||
let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire);
|
||||
if offline_since <= 0 {
|
||||
return None;
|
||||
}
|
||||
let now = current_unix_secs();
|
||||
Some(Duration::from_secs(now.saturating_sub(offline_since as u64)))
|
||||
}
|
||||
|
||||
pub fn mark_failure(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||
let current = self.runtime_state();
|
||||
let now = current_unix_secs();
|
||||
let next = match current {
|
||||
RuntimeDriveHealthState::Online => {
|
||||
self.consecutive_failures.store(1, Ordering::Release);
|
||||
self.consecutive_successes.store(0, Ordering::Release);
|
||||
self.offline_since_unix_secs
|
||||
.compare_exchange(0, now as i64, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.ok();
|
||||
RuntimeDriveHealthState::Suspect
|
||||
}
|
||||
RuntimeDriveHealthState::Suspect => {
|
||||
let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
if failures >= get_drive_suspect_failure_threshold() {
|
||||
RuntimeDriveHealthState::Offline
|
||||
} else {
|
||||
RuntimeDriveHealthState::Suspect
|
||||
}
|
||||
}
|
||||
RuntimeDriveHealthState::Returning => {
|
||||
self.consecutive_failures.store(0, Ordering::Release);
|
||||
self.consecutive_successes.store(0, Ordering::Release);
|
||||
RuntimeDriveHealthState::Offline
|
||||
}
|
||||
RuntimeDriveHealthState::Offline => RuntimeDriveHealthState::Offline,
|
||||
};
|
||||
|
||||
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||
self.transition_state(endpoint, current, next, reason);
|
||||
current == RuntimeDriveHealthState::Online
|
||||
}
|
||||
|
||||
pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||
let current = self.runtime_state();
|
||||
let next = match current {
|
||||
RuntimeDriveHealthState::Online => RuntimeDriveHealthState::Online,
|
||||
RuntimeDriveHealthState::Suspect => RuntimeDriveHealthState::Online,
|
||||
RuntimeDriveHealthState::Offline => {
|
||||
self.consecutive_successes.store(1, Ordering::Release);
|
||||
RuntimeDriveHealthState::Returning
|
||||
}
|
||||
RuntimeDriveHealthState::Returning => {
|
||||
let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
if successes >= get_drive_returning_success_threshold() {
|
||||
RuntimeDriveHealthState::Online
|
||||
} else {
|
||||
RuntimeDriveHealthState::Returning
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let became_online = next == RuntimeDriveHealthState::Online;
|
||||
if became_online {
|
||||
self.status.store(DISK_HEALTH_OK, Ordering::Release);
|
||||
self.consecutive_failures.store(0, Ordering::Release);
|
||||
self.consecutive_successes.store(0, Ordering::Release);
|
||||
}
|
||||
self.transition_state(endpoint, current, next, reason);
|
||||
if became_online {
|
||||
self.log_success();
|
||||
}
|
||||
became_online
|
||||
}
|
||||
|
||||
fn transition_state(
|
||||
&self,
|
||||
endpoint: &Endpoint,
|
||||
current: RuntimeDriveHealthState,
|
||||
next: RuntimeDriveHealthState,
|
||||
reason: &'static str,
|
||||
) {
|
||||
if current == next {
|
||||
return;
|
||||
}
|
||||
|
||||
self.runtime_state.store(next as u32, Ordering::Release);
|
||||
self.last_transition_unix_secs
|
||||
.store(current_unix_secs() as i64, Ordering::Release);
|
||||
|
||||
if matches!(
|
||||
next,
|
||||
RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Offline | RuntimeDriveHealthState::Returning
|
||||
) && self.offline_since_unix_secs.load(Ordering::Acquire) == 0
|
||||
{
|
||||
self.offline_since_unix_secs
|
||||
.store(current_unix_secs() as i64, Ordering::Release);
|
||||
}
|
||||
|
||||
if next == RuntimeDriveHealthState::Online {
|
||||
if let Some(duration) = self.offline_duration() {
|
||||
record_drive_offline_duration(endpoint, duration);
|
||||
record_drive_recovery_class(classify_drive_recovery(duration));
|
||||
}
|
||||
self.offline_since_unix_secs.store(0, Ordering::Release);
|
||||
} else if let Some(duration) = self.offline_duration() {
|
||||
record_drive_offline_duration(endpoint, duration);
|
||||
}
|
||||
|
||||
record_drive_state_transition(endpoint, current, next, reason);
|
||||
record_drive_runtime_state(endpoint, next);
|
||||
}
|
||||
|
||||
/// Increment waiting operations counter
|
||||
pub fn increment_waiting(&self) {
|
||||
self.waiting.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -138,6 +327,13 @@ impl DiskHealthTracker {
|
||||
}
|
||||
}
|
||||
|
||||
fn current_unix_secs() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
impl Default for DiskHealthTracker {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
@@ -187,19 +383,34 @@ impl LocalDiskWrapper {
|
||||
let env_health_check =
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
|
||||
|
||||
Self {
|
||||
let wrapper = Self {
|
||||
disk,
|
||||
health: Arc::new(DiskHealthTracker::new()),
|
||||
health_check: health_check && env_health_check,
|
||||
cancel_token: CancellationToken::new(),
|
||||
disk_id: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
};
|
||||
record_drive_runtime_state(&wrapper.disk.endpoint(), RuntimeDriveHealthState::Online);
|
||||
wrapper
|
||||
}
|
||||
|
||||
pub fn get_disk(&self) -> Arc<LocalDisk> {
|
||||
self.disk.clone()
|
||||
}
|
||||
|
||||
pub fn runtime_state(&self) -> RuntimeDriveHealthState {
|
||||
self.health.runtime_state()
|
||||
}
|
||||
|
||||
pub fn offline_duration_secs(&self) -> Option<u64> {
|
||||
self.health.offline_duration().map(|duration| duration.as_secs())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
|
||||
self.health.force_runtime_state_for_test(state);
|
||||
}
|
||||
|
||||
/// Enable health monitoring after disk creation.
|
||||
/// Used to defer health checks until after startup format loading completes.
|
||||
pub fn enable_health_check(&self) {
|
||||
@@ -220,6 +431,20 @@ impl LocalDiskWrapper {
|
||||
self.cancel_token.cancel();
|
||||
}
|
||||
|
||||
fn spawn_recovery_monitor_if_needed(&self) {
|
||||
if !self.health_check {
|
||||
return;
|
||||
}
|
||||
|
||||
self.health.increment_waiting();
|
||||
let health = Arc::clone(&self.health);
|
||||
let disk = Arc::clone(&self.disk);
|
||||
let cancel_token = self.cancel_token.clone();
|
||||
tokio::spawn(async move {
|
||||
Self::monitor_disk_status(disk, health, cancel_token).await;
|
||||
});
|
||||
}
|
||||
|
||||
/// Monitor disk writability periodically
|
||||
async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
|
||||
// TODO: config interval
|
||||
@@ -257,7 +482,9 @@ impl LocalDiskWrapper {
|
||||
|
||||
|
||||
let test_obj = format!("health-check-{}", Uuid::new_v4());
|
||||
if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err() && health.swap_ok_to_faulty() {
|
||||
if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err()
|
||||
&& health.mark_failure(&disk.endpoint(), "active_health_check_failed")
|
||||
{
|
||||
// Health check failed, disk is considered faulty
|
||||
warn!("health check: failed, disk is considered faulty");
|
||||
|
||||
@@ -346,9 +573,9 @@ impl LocalDiskWrapper {
|
||||
|
||||
/// Monitor disk status and try to bring it back online
|
||||
async fn monitor_disk_status(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
|
||||
const CHECK_EVERY: Duration = Duration::from_secs(5);
|
||||
let check_every = get_drive_returning_probe_interval();
|
||||
|
||||
let mut interval = time::interval(CHECK_EVERY);
|
||||
let mut interval = time::interval(check_every);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -363,12 +590,18 @@ impl LocalDiskWrapper {
|
||||
let test_obj = format!("health-check-{}", Uuid::new_v4());
|
||||
match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await {
|
||||
Ok(_) => {
|
||||
let state_before = health.runtime_state();
|
||||
let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
|
||||
info!("Disk {} recovery probe succeeded; state={:?}", disk.to_string(), state_before);
|
||||
if !is_online {
|
||||
continue;
|
||||
}
|
||||
info!("Disk {} is back online", disk.to_string());
|
||||
health.set_ok();
|
||||
health.decrement_waiting();
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
health.mark_failure(&disk.endpoint(), "recovery_probe_failed");
|
||||
warn!("Disk {} still faulty: {:?}", disk.to_string(), e);
|
||||
}
|
||||
}
|
||||
@@ -436,6 +669,19 @@ impl LocalDiskWrapper {
|
||||
/// Track disk health for an operation.
|
||||
/// This method should wrap disk operations to ensure health checking.
|
||||
pub async fn track_disk_health<T, F, Fut>(&self, operation: F, timeout_duration: Duration) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
self.track_disk_health_with_op("unknown", operation, timeout_duration).await
|
||||
}
|
||||
|
||||
pub async fn track_disk_health_with_op<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
@@ -480,7 +726,21 @@ impl LocalDiskWrapper {
|
||||
Err(_) => {
|
||||
// Timeout occurred, mark disk as potentially faulty and decrement waiting counter
|
||||
self.health.decrement_waiting();
|
||||
warn!("disk operation timeout after {:?}", timeout_duration);
|
||||
if self.health.mark_failure(&self.endpoint(), "operation_timeout") {
|
||||
self.spawn_recovery_monitor_if_needed();
|
||||
}
|
||||
counter!(
|
||||
"rustfs_drive_op_timeout_total",
|
||||
"endpoint" => self.endpoint().to_string(),
|
||||
"op" => op.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
warn!(
|
||||
endpoint = %self.endpoint(),
|
||||
op,
|
||||
timeout_ms = timeout_duration.as_millis(),
|
||||
"Local disk operation timed out"
|
||||
);
|
||||
Err(DiskError::other(format!("disk operation timeout after {timeout_duration:?}")))
|
||||
}
|
||||
}
|
||||
@@ -490,8 +750,12 @@ impl LocalDiskWrapper {
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDiskWrapper {
|
||||
async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
|
||||
self.track_disk_health(|| async { self.disk.read_metadata(volume, path).await }, Duration::ZERO)
|
||||
.await
|
||||
self.track_disk_health_with_op(
|
||||
"read_metadata",
|
||||
|| async { self.disk.read_metadata(volume, path).await },
|
||||
get_drive_metadata_timeout(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn start_scan(&self) -> ScanGuard {
|
||||
@@ -566,15 +830,22 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
|
||||
let result = self.disk.disk_info(opts).await?;
|
||||
self.track_disk_health_with_op(
|
||||
"disk_info",
|
||||
|| async {
|
||||
let result = self.disk.disk_info(opts).await?;
|
||||
|
||||
if let Some(current_disk_id) = *self.disk_id.read().await
|
||||
&& Some(current_disk_id) != result.id
|
||||
{
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
if let Some(current_disk_id) = *self.disk_id.read().await
|
||||
&& Some(current_disk_id) != result.id
|
||||
{
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
|
||||
Ok(result)
|
||||
Ok(result)
|
||||
},
|
||||
get_drive_disk_info_timeout(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
@@ -588,7 +859,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
|
||||
self.track_disk_health(|| async { self.disk.list_volumes().await }, Duration::ZERO)
|
||||
self.track_disk_health_with_op("list_volumes", || async { self.disk.list_volumes().await }, Duration::ZERO)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -603,7 +874,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn walk_dir<W: tokio::io::AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
|
||||
self.track_disk_health(|| async { self.disk.walk_dir(opts, wr).await }, Duration::ZERO)
|
||||
self.track_disk_health_with_op("walk_dir", || async { self.disk.walk_dir(opts, wr).await }, get_drive_walkdir_timeout())
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -711,9 +982,10 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
|
||||
self.track_disk_health(
|
||||
self.track_disk_health_with_op(
|
||||
"list_dir",
|
||||
|| async { self.disk.list_dir(origvolume, volume, dir_path, count).await },
|
||||
get_max_timeout_duration(),
|
||||
get_drive_list_dir_timeout(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -739,14 +1011,6 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<BoxChunkStream> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.read_file_chunks(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn append_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileWriter> {
|
||||
self.track_disk_health(|| async { self.disk.append_file(volume, path).await }, Duration::ZERO)
|
||||
.await
|
||||
@@ -811,3 +1075,74 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::health_state::RuntimeDriveHealthState;
|
||||
|
||||
#[test]
|
||||
fn drive_metadata_timeout_uses_default_when_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
assert_eq!(
|
||||
get_drive_metadata_timeout(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_metadata_timeout_uses_legacy_fallback_when_canonical_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
|
||||
assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(17));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_metadata_timeout_prefers_canonical_over_legacy() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, Some("7"), || {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
|
||||
assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(7));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_state_transitions_from_online_to_suspect_then_offline() {
|
||||
let endpoint = Endpoint::try_from("/tmp/runtime-state-disk").expect("endpoint should parse");
|
||||
let health = DiskHealthTracker::new();
|
||||
|
||||
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert!(health.mark_failure(&endpoint, "timeout"));
|
||||
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
|
||||
assert!(!health.mark_failure(&endpoint, "timeout"));
|
||||
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
|
||||
assert!(health.offline_duration().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_state_transitions_back_online_after_recovery_threshold() {
|
||||
let endpoint = Endpoint::try_from("/tmp/runtime-state-recovery").expect("endpoint should parse");
|
||||
let health = DiskHealthTracker::new();
|
||||
|
||||
health.mark_failure(&endpoint, "timeout");
|
||||
health.mark_failure(&endpoint, "timeout");
|
||||
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
|
||||
|
||||
assert!(!health.mark_recovery_success(&endpoint, "probe"));
|
||||
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
|
||||
|
||||
assert!(!health.mark_recovery_success(&endpoint, "probe"));
|
||||
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
|
||||
|
||||
assert!(health.mark_recovery_success(&endpoint, "probe"));
|
||||
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert!(health.offline_duration().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -724,10 +724,7 @@ mod tests {
|
||||
let path = PathBuf::from("/test/path");
|
||||
let io_error = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "permission denied");
|
||||
|
||||
let context_error = FileAccessDeniedWithContext {
|
||||
path: path.clone(),
|
||||
source: io_error,
|
||||
};
|
||||
let context_error = FileAccessDeniedWithContext { path, source: io_error };
|
||||
|
||||
let display_str = format!("{context_error}");
|
||||
assert!(display_str.contains("/test/path"));
|
||||
|
||||
@@ -67,7 +67,7 @@ pub fn reduce_errs(errors: &[Option<Error>], ignored_errs: &[Error]) -> (usize,
|
||||
let (best_err, best_count) = err_counts
|
||||
.into_iter()
|
||||
.max_by(|(_, c1), (_, c2)| c1.cmp(c2))
|
||||
.unwrap_or((nil_error.clone(), 0));
|
||||
.unwrap_or((nil_error, 0));
|
||||
|
||||
// Compare nil errors with the top non-nil error and prefer the nil error
|
||||
if nil_count > best_count || (nil_count == best_count && nil_count > 0) {
|
||||
@@ -112,7 +112,7 @@ mod tests {
|
||||
fn test_reduce_errs_basic() {
|
||||
let e1 = err_io("a");
|
||||
let e2 = err_io("b");
|
||||
let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
|
||||
let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2), None];
|
||||
let ignored = vec![];
|
||||
let (count, err) = reduce_errs(&errors, &ignored);
|
||||
assert_eq!(count, 2);
|
||||
@@ -124,7 +124,7 @@ mod tests {
|
||||
let e1 = err_io("a");
|
||||
let e2 = err_io("b");
|
||||
let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), Some(e2.clone()), None];
|
||||
let ignored = vec![e2.clone()];
|
||||
let ignored = vec![e2];
|
||||
let (count, err) = reduce_errs(&errors, &ignored);
|
||||
assert_eq!(count, 2);
|
||||
assert_eq!(err, Some(e1));
|
||||
@@ -134,7 +134,7 @@ mod tests {
|
||||
fn test_reduce_quorum_errs() {
|
||||
let e1 = err_io("a");
|
||||
let e2 = err_io("b");
|
||||
let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
|
||||
let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2), None];
|
||||
let ignored = vec![];
|
||||
let quorum_err = Error::FaultyDisk;
|
||||
// quorum = 2, should return e1
|
||||
@@ -167,7 +167,7 @@ mod tests {
|
||||
fn test_reduce_errs_nil_tiebreak() {
|
||||
// Error::Nil and another error have the same count, should prefer Nil
|
||||
let e1 = err_io("a");
|
||||
let errors = vec![Some(e1.clone()), None, Some(e1.clone()), None]; // e1:2, Nil:2
|
||||
let errors = vec![Some(e1.clone()), None, Some(e1), None]; // e1:2, Nil:2
|
||||
let ignored = vec![];
|
||||
let (count, err) = reduce_errs(&errors, &ignored);
|
||||
assert_eq!(count, 2);
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{DiskAPI, DiskStore};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use metrics::{counter, gauge};
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u32)]
|
||||
pub enum RuntimeDriveHealthState {
|
||||
Online = 0,
|
||||
Suspect = 1,
|
||||
Offline = 2,
|
||||
Returning = 3,
|
||||
}
|
||||
|
||||
impl RuntimeDriveHealthState {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Online => "online",
|
||||
Self::Suspect => "suspect",
|
||||
Self::Offline => "offline",
|
||||
Self::Returning => "returning",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_u32(value: u32) -> Self {
|
||||
match value {
|
||||
1 => Self::Suspect,
|
||||
2 => Self::Offline,
|
||||
3 => Self::Returning,
|
||||
_ => Self::Online,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_snapshot_eligible(self) -> bool {
|
||||
matches!(self, Self::Online | Self::Suspect | Self::Returning)
|
||||
}
|
||||
|
||||
pub fn is_strictly_online(self) -> bool {
|
||||
matches!(self, Self::Online)
|
||||
}
|
||||
|
||||
pub fn should_probe_for_admin(self) -> bool {
|
||||
matches!(self, Self::Online | Self::Returning)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DriveRecoveryClass {
|
||||
ShortOffline,
|
||||
MediumOffline,
|
||||
LongOffline,
|
||||
}
|
||||
|
||||
impl DriveRecoveryClass {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::ShortOffline => "short_offline",
|
||||
Self::MediumOffline => "medium_offline",
|
||||
Self::LongOffline => "long_offline",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_drive_suspect_failure_threshold() -> u32 {
|
||||
rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD,
|
||||
rustfs_config::DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD,
|
||||
) as u32
|
||||
}
|
||||
|
||||
pub fn get_drive_returning_success_threshold() -> u32 {
|
||||
rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD,
|
||||
rustfs_config::DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD,
|
||||
) as u32
|
||||
}
|
||||
|
||||
pub fn get_drive_returning_probe_interval() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn get_drive_offline_grace_period() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn get_drive_long_offline_threshold() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn classify_drive_recovery(duration: Duration) -> DriveRecoveryClass {
|
||||
if duration <= get_drive_offline_grace_period() {
|
||||
DriveRecoveryClass::ShortOffline
|
||||
} else if duration >= get_drive_long_offline_threshold() {
|
||||
DriveRecoveryClass::LongOffline
|
||||
} else {
|
||||
DriveRecoveryClass::MediumOffline
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_drive_runtime_state(endpoint: &Endpoint, state: RuntimeDriveHealthState) {
|
||||
let endpoint_label = endpoint.to_string();
|
||||
let pool_label = endpoint.pool_idx.to_string();
|
||||
let set_label = endpoint.set_idx.to_string();
|
||||
let disk_label = endpoint.disk_idx.to_string();
|
||||
|
||||
for candidate in [
|
||||
RuntimeDriveHealthState::Online,
|
||||
RuntimeDriveHealthState::Suspect,
|
||||
RuntimeDriveHealthState::Offline,
|
||||
RuntimeDriveHealthState::Returning,
|
||||
] {
|
||||
gauge!(
|
||||
"rustfs_drive_runtime_state",
|
||||
"endpoint" => endpoint_label.clone(),
|
||||
"pool" => pool_label.clone(),
|
||||
"set" => set_label.clone(),
|
||||
"disk" => disk_label.clone(),
|
||||
"state" => candidate.as_str().to_string()
|
||||
)
|
||||
.set(if candidate == state { 1.0 } else { 0.0 });
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_drive_state_transition(
|
||||
endpoint: &Endpoint,
|
||||
from: RuntimeDriveHealthState,
|
||||
to: RuntimeDriveHealthState,
|
||||
reason: &'static str,
|
||||
) {
|
||||
counter!(
|
||||
"rustfs_drive_state_transition_total",
|
||||
"endpoint" => endpoint.to_string(),
|
||||
"pool" => endpoint.pool_idx.to_string(),
|
||||
"set" => endpoint.set_idx.to_string(),
|
||||
"disk" => endpoint.disk_idx.to_string(),
|
||||
"from" => from.as_str().to_string(),
|
||||
"to" => to.as_str().to_string(),
|
||||
"reason" => reason.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_drive_recovery_class(class: DriveRecoveryClass) {
|
||||
counter!(
|
||||
"rustfs_drive_recovery_class_total",
|
||||
"class" => class.as_str().to_string()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_drive_offline_duration(endpoint: &Endpoint, duration: Duration) {
|
||||
gauge!(
|
||||
"rustfs_drive_offline_duration_seconds",
|
||||
"endpoint" => endpoint.to_string(),
|
||||
"pool" => endpoint.pool_idx.to_string(),
|
||||
"set" => endpoint.set_idx.to_string(),
|
||||
"disk" => endpoint.disk_idx.to_string()
|
||||
)
|
||||
.set(duration.as_secs_f64());
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DriveMembershipSnapshot {
|
||||
pub online: Vec<DiskStore>,
|
||||
pub suspect: Vec<DiskStore>,
|
||||
pub returning: Vec<DiskStore>,
|
||||
pub offline: Vec<DiskStore>,
|
||||
}
|
||||
|
||||
impl DriveMembershipSnapshot {
|
||||
pub fn from_optional_disks(disks: &[Option<DiskStore>]) -> Self {
|
||||
let mut snapshot = Self::default();
|
||||
|
||||
for disk in disks.iter().flatten() {
|
||||
match disk.runtime_state() {
|
||||
RuntimeDriveHealthState::Online => snapshot.online.push(disk.clone()),
|
||||
RuntimeDriveHealthState::Suspect => snapshot.suspect.push(disk.clone()),
|
||||
RuntimeDriveHealthState::Returning => snapshot.returning.push(disk.clone()),
|
||||
RuntimeDriveHealthState::Offline => snapshot.offline.push(disk.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
snapshot
|
||||
}
|
||||
|
||||
pub fn scanner_heal_candidates(&self) -> Vec<DiskStore> {
|
||||
let mut disks = Vec::with_capacity(self.online.len() + self.suspect.len() + self.returning.len());
|
||||
disks.extend(self.online.iter().cloned());
|
||||
disks.extend(self.suspect.iter().cloned());
|
||||
disks.extend(self.returning.iter().cloned());
|
||||
disks
|
||||
}
|
||||
|
||||
pub fn strict_online_candidates(&self) -> Vec<DiskStore> {
|
||||
self.online.clone()
|
||||
}
|
||||
|
||||
pub fn strict_online_local_candidates(&self) -> Vec<DiskStore> {
|
||||
self.online.iter().filter(|disk| disk.is_local()).cloned().collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn runtime_drive_health_state_snapshot_eligibility_matches_membership_policy() {
|
||||
assert!(RuntimeDriveHealthState::Online.is_snapshot_eligible());
|
||||
assert!(RuntimeDriveHealthState::Suspect.is_snapshot_eligible());
|
||||
assert!(RuntimeDriveHealthState::Returning.is_snapshot_eligible());
|
||||
assert!(!RuntimeDriveHealthState::Offline.is_snapshot_eligible());
|
||||
}
|
||||
}
|
||||
+251
-799
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user