mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-30 10:08:58 +00:00
Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 07e643cac2 | |||
| 61dbaf60d3 | |||
| c82ee6be58 | |||
| 48b328d0d2 | |||
| 7b3d9e0c04 | |||
| 73f603b625 | |||
| 3a4937367a | |||
| 513e5f1018 | |||
| 56179210ab | |||
| f3a7a4b0da | |||
| 0fa6dc5946 | |||
| b3a781bce0 | |||
| 49e04bf343 | |||
| 299b739f9f | |||
| 53270054e2 | |||
| 978ac13eb5 | |||
| 35c5693486 | |||
| 8f15dd2cef | |||
| 5ef2731a6b | |||
| 081c10f073 | |||
| 28ee1e5e1a |
@@ -152,7 +152,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the nightly profile derives its set as "the replication module MINUS this
|
||||
# allowlist", so any new replication test lands in nightly by default (never
|
||||
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
||||
# regexes byte-identical. Count invariant: 20 here + 23 nightly = 43 total
|
||||
# regexes byte-identical. Count invariant: 20 here + 27 nightly = 47 total
|
||||
# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
|
||||
# HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
|
||||
# (#4724) because they set a loopback (127.0.0.1) replication target that the
|
||||
@@ -202,10 +202,14 @@ fail-fast = false
|
||||
# backlog#1147 repl-1 (deps: ci-4). Runs the SLOW / cross-process replication
|
||||
# tests that are unfit for the per-PR e2e-smoke gate:
|
||||
#
|
||||
# * 8 bucket-replication data-plane tests — they PUT/delete objects and poll
|
||||
# until source and target converge; two replicate over HTTPS.
|
||||
# * 2 remote-target TLS validation tests.
|
||||
# * 12 bucket-replication data-plane/helper tests — they PUT/delete objects
|
||||
# and poll until source and target converge; two replicate over HTTPS, two
|
||||
# pin active SSE failure contracts, and one guards event/history observers.
|
||||
# The SSE-S3 contract remains ignored under backlog#1291.
|
||||
# * 11 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
|
||||
# servers and drives the cross-process site-replication control plane.
|
||||
# * 1 `_real_three_node` site-replication test.
|
||||
# * 1 `_real_single_node` service-account round-trip test.
|
||||
#
|
||||
# The set is defined as "everything in replication_extension_test that is NOT
|
||||
@@ -261,7 +265,7 @@ path = "junit.xml"
|
||||
# object_lambda) — too heavy for the merge budget; they run in ci-7's
|
||||
# nightly 4-node lane.
|
||||
# * replication_extension_test — repl-1 already splits it into the PR
|
||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (20 slow) lanes and reserves
|
||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (27 slow) lanes and reserves
|
||||
# it for those, so e2e-full does not double-run it.
|
||||
# * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
|
||||
# manual-localhost:9000 reliant/policy tests are ci-13's migration.
|
||||
|
||||
@@ -56,6 +56,7 @@ runs:
|
||||
libssl-dev \
|
||||
ripgrep \
|
||||
unzip \
|
||||
zip \
|
||||
protobuf-compiler
|
||||
|
||||
- name: Install protoc
|
||||
|
||||
@@ -223,7 +223,7 @@ jobs:
|
||||
needs: [ build-check, prepare-platform-matrix ]
|
||||
if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 90
|
||||
timeout-minutes: 150
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
# Release binaries ship without dial9 telemetry and therefore do not need
|
||||
@@ -439,11 +439,11 @@ jobs:
|
||||
# Release/Prerelease build: rustfs-${platform}-${arch}-${variant}-v${version}.zip
|
||||
PACKAGE_NAME="${PACKAGE_BASENAME}-v${PACKAGE_VERSION}"
|
||||
fi
|
||||
|
||||
|
||||
# Create zip packages for all platforms
|
||||
# Ensure zip is available
|
||||
if ! command -v zip &> /dev/null; then
|
||||
if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
|
||||
if [[ "${{ matrix.os }}" == "ubuntu-latest" || "${{ matrix.platform }}" == "linux" ]]; then
|
||||
sudo apt-get update && sudo apt-get install -y zip
|
||||
fi
|
||||
fi
|
||||
@@ -583,9 +583,19 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# The self-hosted Linux runners do not ship the aws CLI (GitHub-hosted
|
||||
# images did). Install it on demand so R2 uploads survive a fresh runner
|
||||
# instead of hard-failing here.
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
echo "❌ aws CLI not found on runner; cannot upload to R2"
|
||||
exit 1
|
||||
echo "aws CLI not found on runner; installing..."
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
elif command -v brew >/dev/null 2>&1; then
|
||||
brew install awscli
|
||||
else
|
||||
echo "❌ aws CLI missing and no apt-get/brew to install it; cannot upload to R2"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
|
||||
@@ -557,6 +557,14 @@ jobs:
|
||||
- name: Setup Rust toolchain for s3s-e2e installation
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
|
||||
# The sm-standard-* custom runner images (introduced in #4884) ship no C
|
||||
# toolchain, unlike GitHub-hosted ubuntu-latest. Installing s3s-e2e below
|
||||
# compiles it from source on a cache miss, and build scripts need cc.
|
||||
- name: Install build tools for s3s-e2e compilation
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential cmake pkg-config libssl-dev
|
||||
|
||||
- name: Install s3s-e2e test tool
|
||||
uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
|
||||
with:
|
||||
|
||||
@@ -15,20 +15,24 @@
|
||||
# Nightly full replication e2e lane (backlog#1147 repl-1, deps: ci-4).
|
||||
#
|
||||
# The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the 20
|
||||
# FAST bucket-replication tests. This scheduled lane runs the remaining 18
|
||||
# FAST replication tests. This scheduled lane runs the remaining 27
|
||||
# heavier replication e2e tests that are unfit for a per-PR gate:
|
||||
#
|
||||
# * 8 bucket-replication data-plane tests (PUT/delete + poll for convergence;
|
||||
# two replicate over HTTPS).
|
||||
# * 9 `_real_dual_node` site-replication tests (each spawns TWO rustfs
|
||||
# * 2 remote-target TLS validation tests.
|
||||
# * 12 bucket-replication data-plane/helper tests (PUT/delete + poll for
|
||||
# convergence; two replicate over HTTPS, two pin active SSE failure
|
||||
# contracts, and one guards event/history observers). The SSE-S3 contract
|
||||
# remains ignored under backlog#1291.
|
||||
# * 11 `_real_dual_node` site-replication tests (each spawns TWO rustfs
|
||||
# servers and drives the cross-process site-replication control plane).
|
||||
# * 1 `_real_three_node` site-replication test.
|
||||
# * 1 `_real_single_node` service-account round-trip test.
|
||||
#
|
||||
# The selection is the [profile.e2e-repl-nightly] default-filter in
|
||||
# .config/nextest.toml — the single wiring mechanism (repl-1 / ci-4). Do NOT
|
||||
# add ad-hoc cargo-test steps here; change the filterset instead.
|
||||
#
|
||||
# Explicit division of labor: these 18 tests run ONLY here, never double-run
|
||||
# Explicit division of labor: these 27 tests run ONLY here, never double-run
|
||||
# in ci-5's future e2e-full merge gate. TODO(ci-7): once the ci domain's
|
||||
# consolidated scheduled e2e workflow exists, fold this interim repl-owned lane
|
||||
# into it rather than growing a second scheduled entrypoint.
|
||||
@@ -80,7 +84,12 @@ jobs:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: python3 -m pip install --user --upgrade pip awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip awscurl
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify awscurl
|
||||
run: test -x "$AWSCURL_PATH"
|
||||
|
||||
# Build the rustfs binary once up front. The e2e tests spawn it as a
|
||||
# child process (crates/e2e_test/src/common.rs) and will build it on
|
||||
|
||||
Generated
+93
-137
@@ -319,17 +319,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"blake2 0.11.0-rc.6",
|
||||
"blake2",
|
||||
"cpufeatures 0.3.0",
|
||||
"password-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrayref"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.8"
|
||||
@@ -1503,22 +1497,13 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.0"
|
||||
version = "2.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
|
||||
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blake2"
|
||||
version = "0.10.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
|
||||
dependencies = [
|
||||
"digest 0.10.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blake2"
|
||||
version = "0.11.0-rc.6"
|
||||
@@ -1528,20 +1513,6 @@ dependencies = [
|
||||
"digest 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blake3"
|
||||
version = "1.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"arrayvec",
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"constant_time_eq",
|
||||
"cpufeatures 0.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
@@ -1908,9 +1879,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.1"
|
||||
version = "4.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
|
||||
checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
@@ -1918,9 +1889,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.0"
|
||||
version = "4.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
|
||||
checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
@@ -2628,7 +2599,6 @@ dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
"async-trait",
|
||||
"bzip2",
|
||||
"chrono",
|
||||
"datafusion-catalog",
|
||||
"datafusion-catalog-listing",
|
||||
@@ -2655,11 +2625,9 @@ dependencies = [
|
||||
"datafusion-physical-plan",
|
||||
"datafusion-session",
|
||||
"datafusion-sql",
|
||||
"flate2",
|
||||
"futures",
|
||||
"indexmap 2.14.0",
|
||||
"itertools 0.14.0",
|
||||
"liblzma",
|
||||
"log",
|
||||
"object_store",
|
||||
"parking_lot",
|
||||
@@ -2669,7 +2637,6 @@ dependencies = [
|
||||
"tokio",
|
||||
"url",
|
||||
"uuid",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2759,10 +2726,8 @@ version = "54.0.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=dae03ee062b2abf986de8df12ea82fb1578a2d99#dae03ee062b2abf986de8df12ea82fb1578a2d99"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-compression",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"bzip2",
|
||||
"chrono",
|
||||
"datafusion-common",
|
||||
"datafusion-common-runtime",
|
||||
@@ -2773,19 +2738,15 @@ dependencies = [
|
||||
"datafusion-physical-expr-common",
|
||||
"datafusion-physical-plan",
|
||||
"datafusion-session",
|
||||
"flate2",
|
||||
"futures",
|
||||
"glob",
|
||||
"itertools 0.14.0",
|
||||
"liblzma",
|
||||
"log",
|
||||
"object_store",
|
||||
"parking_lot",
|
||||
"rand 0.9.5",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"url",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2953,8 +2914,6 @@ dependencies = [
|
||||
"arrow",
|
||||
"arrow-buffer",
|
||||
"base64 0.22.1",
|
||||
"blake2 0.10.6",
|
||||
"blake3",
|
||||
"chrono",
|
||||
"chrono-tz",
|
||||
"datafusion-common",
|
||||
@@ -2967,12 +2926,10 @@ dependencies = [
|
||||
"hex",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"md-5 0.11.0",
|
||||
"memchr",
|
||||
"num-traits",
|
||||
"rand 0.9.5",
|
||||
"regex",
|
||||
"sha2 0.11.0",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -3652,7 +3609,7 @@ version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -3687,7 +3644,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -3704,6 +3661,9 @@ dependencies = [
|
||||
"flate2",
|
||||
"futures",
|
||||
"http 1.4.2",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"local-ip-address",
|
||||
"md5",
|
||||
"rand 0.10.2",
|
||||
@@ -4086,7 +4046,7 @@ version = "25.12.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
@@ -5084,7 +5044,6 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tower-service",
|
||||
"webpki-roots 1.0.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5332,7 +5291,7 @@ version = "0.7.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
@@ -5836,7 +5795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9f8ff371890db2cf65a0758dba9a79f9cd965de369f6dbdc6581a22780af45e"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
@@ -6392,7 +6351,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"btoi",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -6424,7 +6383,7 @@ version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"byteorder",
|
||||
"derive_builder",
|
||||
"getset",
|
||||
@@ -6472,7 +6431,7 @@ version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6485,7 +6444,7 @@ version = "0.30.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6497,7 +6456,7 @@ version = "0.31.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6675,7 +6634,7 @@ version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"libloading",
|
||||
"nvml-wrapper-sys",
|
||||
"static_assertions",
|
||||
@@ -6726,7 +6685,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"dispatch2",
|
||||
"objc2",
|
||||
]
|
||||
@@ -6743,7 +6702,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -7270,7 +7229,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97f6fccfd2d9d2df765ca23ff85fe5cc437fb0e6d3e164e4d3cbe09d14780c93"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"thiserror 2.0.18",
|
||||
"zerocopy",
|
||||
"zerocopy-derive",
|
||||
@@ -7782,7 +7741,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
|
||||
dependencies = [
|
||||
"bit-set",
|
||||
"bit-vec 0.8.0",
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"num-traits",
|
||||
"rand 0.9.5",
|
||||
"rand_chacha 0.9.0",
|
||||
@@ -7924,7 +7883,7 @@ version = "0.13.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"memchr",
|
||||
"unicase",
|
||||
]
|
||||
@@ -8255,7 +8214,7 @@ version = "11.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8364,7 +8323,7 @@ version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8435,9 +8394,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.0"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
|
||||
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -8447,9 +8406,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.15"
|
||||
version = "0.4.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db"
|
||||
checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -8678,7 +8637,7 @@ checksum = "ca6c3c1dd0a9ad3a9915a2f6e907d158f9a7e9ac32ddc772b003faafc027d9a8"
|
||||
dependencies = [
|
||||
"aes 0.9.1",
|
||||
"aws-lc-rs",
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"block-padding 0.4.2",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -8760,7 +8719,7 @@ version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed8949eca4163c18a8f59ff96d32cf61e9c13b9735e21ef32b3907f4aafa1a9"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
@@ -8845,7 +8804,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -8865,7 +8824,6 @@ dependencies = [
|
||||
"chrono",
|
||||
"clap",
|
||||
"const-str",
|
||||
"criterion",
|
||||
"datafusion",
|
||||
"flatbuffers",
|
||||
"futures",
|
||||
@@ -8977,7 +8935,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -8999,7 +8957,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9014,7 +8972,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"metrics",
|
||||
@@ -9029,7 +8987,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"insta",
|
||||
"rustfs-io-core",
|
||||
@@ -9041,7 +8999,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"serde",
|
||||
@@ -9050,7 +9008,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9063,7 +9021,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9083,7 +9041,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"path-clean",
|
||||
@@ -9094,7 +9052,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"async-channel",
|
||||
@@ -9228,7 +9186,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9237,7 +9195,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"byteorder",
|
||||
@@ -9263,7 +9221,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -9287,14 +9245,13 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9331,7 +9288,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"memmap2",
|
||||
@@ -9343,7 +9300,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"metrics",
|
||||
@@ -9406,7 +9363,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -9432,7 +9389,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -9463,7 +9420,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"proptest",
|
||||
@@ -9483,7 +9440,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"crossbeam-queue",
|
||||
@@ -9504,7 +9461,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"humantime",
|
||||
@@ -9518,7 +9475,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9550,7 +9507,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -9568,7 +9525,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -9584,7 +9541,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -9636,7 +9593,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -9665,7 +9622,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -9724,7 +9681,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"prost 0.14.4",
|
||||
@@ -9742,7 +9699,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -9759,7 +9716,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -9797,7 +9754,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -9819,14 +9776,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"rustfs-s3-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9834,7 +9791,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -9861,7 +9818,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
@@ -9877,7 +9834,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -9909,14 +9866,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9932,7 +9889,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"insta",
|
||||
@@ -9946,7 +9903,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -9996,7 +9953,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"rustfs-ecstore",
|
||||
"rustfs-storage-api",
|
||||
@@ -10008,7 +9965,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"metrics",
|
||||
@@ -10028,7 +9985,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -10064,10 +10021,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2 0.11.0-rc.6",
|
||||
"blake2",
|
||||
"brotli",
|
||||
"bytes",
|
||||
"convert_case 0.11.0",
|
||||
@@ -10103,7 +10060,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10164,7 +10121,7 @@ version = "0.38.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.4.15",
|
||||
@@ -10177,7 +10134,7 @@ version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
@@ -10490,7 +10447,7 @@ version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"core-foundation 0.10.1",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
@@ -10980,9 +10937,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "snap"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
|
||||
checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
@@ -11141,9 +11098,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "starshard"
|
||||
version = "2.2.1"
|
||||
version = "2.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4dafd0cbefb050fa7a7f95ac229f7978ed4c99866a1a5b62e34a154b52d73d77"
|
||||
checksum = "5d298eb1bb81d6e5ddf447f3d26698d6ac5e5b9502b03004dbc2a2e2f8b572b6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hashbrown 0.17.1",
|
||||
@@ -11364,7 +11321,7 @@ version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"core-foundation 0.9.4",
|
||||
"system-configuration-sys",
|
||||
]
|
||||
@@ -11415,7 +11372,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.59.0",
|
||||
@@ -11642,7 +11599,6 @@ dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
"mio",
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"socket2",
|
||||
@@ -11896,7 +11852,7 @@ version = "0.6.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
@@ -11915,7 +11871,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"bitflags 2.13.0",
|
||||
"bitflags 2.13.1",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
@@ -12254,9 +12210,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.23.5"
|
||||
version = "1.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a"
|
||||
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
|
||||
+108
-108
@@ -68,7 +68,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.96.0"
|
||||
version = "1.0.0-beta.9"
|
||||
version = "1.0.0-beta.10-preview.1"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -85,56 +85,56 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.9" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.9" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.9" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.9" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.9" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.9" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.9" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.9" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.9" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.9" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.9" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.9" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.9" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.9" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.9" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.9" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.9" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.9" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.9" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.9" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.9" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.9" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.9" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.9" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.9" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.9" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.9" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.9" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.9" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.9" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.9" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.9" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.9" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.9" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.9" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.9" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.9" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.9" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.9" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.9" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.9" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.9" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.9" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.9" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.9" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.10-preview.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.10-preview.1" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { version = "0.0.18", default-features = false, features = ["tokio", "deflate"] }
|
||||
mysql_async = { version = "0.37", default-features = false, features = ["default-rustls", "tracing"] }
|
||||
async_zip = { default-features = false, version = "0.0.18" }
|
||||
mysql_async = { default-features = false, version = "0.37" }
|
||||
async-compression = { version = "0.4.42" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.89"
|
||||
@@ -145,32 +145,32 @@ futures-core = "0.3.32"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.32"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { version = "6.8.0", default-features = false, features = ["tokio-rustls-runtime", "telemetry"] }
|
||||
lapin = { version = "4.10.0", default-features = false, features = ["tokio", "rustls", "rustls--aws_lc_rs"] }
|
||||
hyper = { version = "1.10.1", features = ["http2", "http1", "server"] }
|
||||
hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
|
||||
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
pulsar = { default-features = false, version = "6.8.0" }
|
||||
lapin = { default-features = false, version = "4.10.0" }
|
||||
hyper = { version = "1.10.1" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.9" }
|
||||
hyper-util = { version = "0.1.20" }
|
||||
http = "1.4.2"
|
||||
http-body = "1.1.0"
|
||||
http-body-util = "0.1.4"
|
||||
minlz = "1.2.3"
|
||||
reqwest = { version = "0.13.4", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
|
||||
reqwest = { default-features = false, version = "0.13.4" }
|
||||
rustfs-kafka-async = { version = "1.2.0" }
|
||||
socket2 = { version = "0.6.5", features = ["all"] }
|
||||
tokio = { version = "1.52.3", features = ["fs", "rt-multi-thread"] }
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.52.3" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.4" }
|
||||
tokio-stream = { version = "0.1.18" }
|
||||
tokio-test = "0.4.5"
|
||||
tokio-util = { version = "0.7.18", features = ["io", "compat"] }
|
||||
tonic = { version = "0.14.6", features = ["gzip", "deflate"] }
|
||||
tokio-util = { version = "0.7.18" }
|
||||
tonic = { version = "0.14.6" }
|
||||
tonic-prost = { version = "0.14.6" }
|
||||
tonic-prost-build = { version = "0.14.6" }
|
||||
tower = { version = "0.5.3", features = ["timeout"] }
|
||||
tower-http = { version = "0.7.0", features = ["cors"] }
|
||||
tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.0" }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = "0.21.0"
|
||||
bytes = { version = "1.12.1", features = ["serde"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
bytesize = "2.4.2"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
@@ -179,8 +179,8 @@ prost = "0.14.4"
|
||||
quick-xml = "0.41.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["raw_value"] }
|
||||
serde = { version = "1.0.228" }
|
||||
serde_json = { version = "1.0.150" }
|
||||
serde_urlencoded = "0.7.1"
|
||||
|
||||
# Cryptography and Security
|
||||
@@ -188,33 +188,33 @@ serde_urlencoded = "0.7.1"
|
||||
# matching stable releases are not available yet, while previous stable lines
|
||||
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
|
||||
# releases.
|
||||
aes-gcm = { version = "=0.11.0", features = ["rand_core"] }
|
||||
aes-gcm = { version = "=0.11.0" }
|
||||
argon2 = { version = "=0.6.0-rc.8" }
|
||||
blake2 = "=0.11.0-rc.6"
|
||||
chacha20poly1305 = { version = "=0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "10.4.0", features = ["aws_lc_rs"] }
|
||||
openidconnect = { version = "4.0", default-features = false, features = ["accept-rfc3339-timestamps"] }
|
||||
jsonwebtoken = { version = "10.4.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { version = "0.23.42", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls = { default-features = false, version = "0.23.42" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.15.0"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.0"
|
||||
subtle = "2.6"
|
||||
zeroize = { version = "1.9.0", features = ["derive"] }
|
||||
zeroize = { version = "1.9.0" }
|
||||
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45", features = ["serde"] }
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.32", features = ["serde"] }
|
||||
time = { version = "0.3.53", features = ["std", "parsing", "formatting", "macros", "serde"] }
|
||||
jiff = { version = "0.2.32" }
|
||||
time = { version = "0.3.53" }
|
||||
|
||||
# Database
|
||||
deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] }
|
||||
tokio-postgres = { version = "0.7.18", default-features = false, features = ["runtime", "with-serde_json-1"] }
|
||||
deadpool-postgres = { version = "0.14" }
|
||||
tokio-postgres = { default-features = false, version = "0.7.18" }
|
||||
tokio-postgres-rustls = "0.14.0"
|
||||
|
||||
# Utilities and Tools
|
||||
@@ -225,22 +225,22 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.9.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-s3 = { version = "1.138.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-smithy-http-client = { version = "1.2.0", default-features = false, features = ["default-client", "rustls-aws-lc"] }
|
||||
aws-smithy-runtime-api = { version = "1.13.0", features = ["http-1x"] }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.138.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||
aws-smithy-runtime-api = { version = "1.13.0" }
|
||||
aws-smithy-types = { version = "1.6.1" }
|
||||
base64 = "0.22.1"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
clap = { version = "4.6.1", features = ["derive", "env"] }
|
||||
const-str = { version = "1.1.0", features = ["std", "proc"] }
|
||||
clap = { version = "4.6.2" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.11.0"
|
||||
criterion = { version = "0.8", features = ["html_reports"] }
|
||||
criterion = { version = "0.8" }
|
||||
crossbeam-queue = "0.3.13"
|
||||
crossbeam-channel = "0.5.16"
|
||||
crossbeam-deque = "0.8.7"
|
||||
crossbeam-utils = "0.8.22"
|
||||
datafusion = { git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
|
||||
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.13"
|
||||
faster-hex = "0.10.0"
|
||||
@@ -248,11 +248,11 @@ flate2 = "1.1.9"
|
||||
glob = "0.3.3"
|
||||
google-cloud-storage = "1.16.0"
|
||||
google-cloud-auth = "1.14.0"
|
||||
hashbrown = { version = "0.17.1", features = ["serde", "rayon"] }
|
||||
hashbrown = { version = "0.17.1" }
|
||||
hex = "0.4.3"
|
||||
hex-simd = "0.8.0"
|
||||
highway = { version = "1.3.0" }
|
||||
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.0"
|
||||
libc = "0.2.186"
|
||||
libsystemd = "0.7.2"
|
||||
@@ -263,7 +263,7 @@ matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
md5 = "0.8.1"
|
||||
mime_guess = "2.0.5"
|
||||
moka = { version = "0.12.15", features = ["future"] }
|
||||
moka = { version = "0.12.15" }
|
||||
netif = "0.1.6"
|
||||
num_cpus = { version = "1.17.0" }
|
||||
nvml-wrapper = "0.12.1"
|
||||
@@ -273,27 +273,27 @@ path-clean = "1.0.1"
|
||||
percent-encoding = "2.3.2"
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.2", features = ["serde"] }
|
||||
rand = { version = "0.10.2" }
|
||||
ratelimit = "0.10.1"
|
||||
rayon = "1.12.0"
|
||||
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.0", features = ["simd-accel"] }
|
||||
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.0" }
|
||||
#reed-solomon-erasure = { version = "6.0", features = ["simd-accel"], git = "https://github.com/houseme/reed-solomon-erasure",rev = "main" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.0" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] }
|
||||
redis = { version = "1.4.0", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
|
||||
rustix = { version = "1.1.4", features = ["fs"] }
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.33.2" }
|
||||
redis = { version = "1.4.0" }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "ce69c3f10824535c7c24b2f71cdb2aaa4dffb5e0", features = ["minio"] }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "ce69c3f10824535c7c24b2f71cdb2aaa4dffb5e0" }
|
||||
serial_test = "3.5.0"
|
||||
shadow-rs = { version = "2.0.0", default-features = false }
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
smallvec = { version = "1.15.2", features = ["serde"] }
|
||||
smallvec = { version = "1.15.2" }
|
||||
smartstring = "1.0.1"
|
||||
snap = "1.1.1"
|
||||
starshard = { version = "2.2.1", features = ["rayon", "async", "serde"] }
|
||||
strum = { version = "0.28.0", features = ["derive"] }
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.2.2" }
|
||||
strum = { version = "0.28.0" }
|
||||
sysinfo = "0.39.6"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
@@ -303,15 +303,15 @@ tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = { version = "0.33" }
|
||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
|
||||
tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.23.5", features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
uuid = { version = "1.24.0" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
walkdir = "2.5.0"
|
||||
windows = { version = "0.62.2" }
|
||||
xxhash-rust = { version = "0.8.17", features = ["xxh64", "xxh3"] }
|
||||
xxhash-rust = { version = "0.8.17" }
|
||||
zip = "8.6.0"
|
||||
zstd = "0.13.3"
|
||||
|
||||
@@ -319,19 +319,19 @@ zstd = "0.13.3"
|
||||
metrics = "0.24.6"
|
||||
dial9-tokio-telemetry = "0.3"
|
||||
opentelemetry = { version = "0.32.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.32.0", features = ["experimental_span_attributes", "experimental_metadata_attributes"] }
|
||||
opentelemetry-otlp = { version = "0.32.0", features = ["gzip-http", "reqwest-rustls"] }
|
||||
opentelemetry_sdk = { version = "0.32.1", features = ["rt-tokio"] }
|
||||
opentelemetry-semantic-conventions = { version = "0.32.1", features = ["semconv_experimental"] }
|
||||
opentelemetry-appender-tracing = { version = "0.32.0" }
|
||||
opentelemetry-otlp = { version = "0.32.0" }
|
||||
opentelemetry_sdk = { version = "0.32.1" }
|
||||
opentelemetry-semantic-conventions = { version = "0.32.1" }
|
||||
opentelemetry-stdout = { version = "0.32.0" }
|
||||
pyroscope = { version = "2.1.0", features = ["backend-pprof-rs"] }
|
||||
pyroscope = { version = "2.1.0" }
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.0", features = ["experimental"] }
|
||||
libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = "0.14.8"
|
||||
russh = { version = "0.62.2", features = ["serde"] }
|
||||
russh = { version = "0.62.2" }
|
||||
russh-sftp = "2.3.0"
|
||||
|
||||
# WebDAV
|
||||
@@ -341,7 +341,7 @@ dav-server = "0.11.0"
|
||||
mimalloc = "0.1"
|
||||
hotpath = "0.21"
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48", features = ["yaml", "json"] }
|
||||
insta = { version = "1.48" }
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
ignored = ["rustfs"]
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.9
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.10-preview.1
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.9
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.10-preview.1
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -27,17 +27,17 @@ categories = ["web-programming", "development-tools", "asynchronous", "api-bindi
|
||||
|
||||
[dependencies]
|
||||
rustfs-targets = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["audit", "constants", "server-config-model"] }
|
||||
rustfs-config = { workspace = true, features = ["audit", "server-config-model"] }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
const-str = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
const-str = { workspace = true, features = ["std", "proc"] }
|
||||
futures = { workspace = true }
|
||||
hashbrown = { workspace = true }
|
||||
hashbrown = { workspace = true, features = ["serde", "rayon"] }
|
||||
metrics = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "time", "macros"] }
|
||||
tracing = { workspace = true, features = ["std", "attributes"] }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -26,14 +26,14 @@ categories = ["web-programming", "development-tools", "network-programming"]
|
||||
documentation = "https://docs.rs/rustfs-checksums/latest/rustfs_checksum/"
|
||||
|
||||
[dependencies]
|
||||
bytes = { workspace = true }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
crc-fast = { workspace = true }
|
||||
http = { workspace = true }
|
||||
base64-simd = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
sha1 = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
xxhash-rust = { workspace = true }
|
||||
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
|
||||
|
||||
[dev-dependencies]
|
||||
pretty_assertions = { workspace = true }
|
||||
|
||||
@@ -28,14 +28,14 @@ categories = ["web-programming", "development-tools", "data-structures"]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
tokio = { workspace = true }
|
||||
tonic = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tonic = { workspace = true, features = ["gzip", "deflate"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
rmp-serde = { workspace = true }
|
||||
s3s = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -13,15 +13,15 @@ categories = ["concurrency", "filesystem"]
|
||||
[dependencies]
|
||||
# Internal crates
|
||||
rustfs-io-core = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
# Async runtime
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
|
||||
|
||||
# Logging
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
insta = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "rt-multi-thread"] }
|
||||
insta = { workspace = true, features = ["yaml", "json"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "rt-multi-thread", "fs"] }
|
||||
|
||||
@@ -25,9 +25,9 @@ keywords = ["configuration", "settings", "management", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "config"]
|
||||
|
||||
[dependencies]
|
||||
const-str = { workspace = true, optional = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
serde_json = { workspace = true, optional = true }
|
||||
const-str = { workspace = true, optional = true, features = ["std", "proc"] }
|
||||
serde = { workspace = true, optional = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, optional = true, features = ["raw_value"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/// Enable or disable per-client rate limiting for the S3 API.
|
||||
///
|
||||
/// When enabled (and `RUSTFS_API_RATE_LIMIT_RPM` > 0), requests are throttled
|
||||
/// per client IP using a token bucket; over-limit requests receive
|
||||
/// `429 Too Many Requests` with a `Retry-After` header. Internode RPC/gRPC,
|
||||
/// health probes, and the console (which has its own limiter) are exempt.
|
||||
/// Environment variable: RUSTFS_API_RATE_LIMIT_ENABLE
|
||||
/// Example: RUSTFS_API_RATE_LIMIT_ENABLE=true
|
||||
pub const ENV_API_RATE_LIMIT_ENABLE: &str = "RUSTFS_API_RATE_LIMIT_ENABLE";
|
||||
|
||||
/// Default for `RUSTFS_API_RATE_LIMIT_ENABLE`.
|
||||
///
|
||||
/// Disabled by default: RustFS ships permissive and operators opt in to
|
||||
/// abuse-protection hardening. When disabled the request path is unchanged.
|
||||
pub const DEFAULT_API_RATE_LIMIT_ENABLE: bool = false;
|
||||
|
||||
/// Sustained S3 API request budget per client IP, in requests per minute.
|
||||
///
|
||||
/// `0` means unlimited (rate limiting stays inert even when enabled).
|
||||
/// Environment variable: RUSTFS_API_RATE_LIMIT_RPM
|
||||
/// Example: RUSTFS_API_RATE_LIMIT_RPM=6000
|
||||
pub const ENV_API_RATE_LIMIT_RPM: &str = "RUSTFS_API_RATE_LIMIT_RPM";
|
||||
|
||||
/// Default for `RUSTFS_API_RATE_LIMIT_RPM`.
|
||||
///
|
||||
/// `0` (unlimited) so that setting only the enable switch cannot throttle
|
||||
/// traffic by surprise; operators must choose an explicit budget.
|
||||
pub const DEFAULT_API_RATE_LIMIT_RPM: u32 = 0;
|
||||
|
||||
/// Burst capacity per client IP (maximum tokens in the bucket).
|
||||
///
|
||||
/// Allows short spikes above the sustained rate. `0` means "same as RPM".
|
||||
/// Environment variable: RUSTFS_API_RATE_LIMIT_BURST
|
||||
/// Example: RUSTFS_API_RATE_LIMIT_BURST=200
|
||||
pub const ENV_API_RATE_LIMIT_BURST: &str = "RUSTFS_API_RATE_LIMIT_BURST";
|
||||
|
||||
/// Default for `RUSTFS_API_RATE_LIMIT_BURST` (`0` = same as RPM).
|
||||
pub const DEFAULT_API_RATE_LIMIT_BURST: u32 = 0;
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod api;
|
||||
pub(crate) mod app;
|
||||
pub(crate) mod body_limits;
|
||||
pub(crate) mod capacity;
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#[cfg(feature = "constants")]
|
||||
pub mod constants;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::api::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::app::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::body_limits::*;
|
||||
|
||||
@@ -27,9 +27,9 @@ categories = ["web-programming", "development-tools", "data-structures", "securi
|
||||
[dependencies]
|
||||
base64-simd = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json.workspace = true
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
sha2 = { workspace = true }
|
||||
time = { workspace = true, features = ["serde", "parsing", "formatting", "macros"] }
|
||||
|
||||
|
||||
@@ -29,23 +29,23 @@ documentation = "https://docs.rs/rustfs-crypto/latest/rustfs_crypto/"
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
aes-gcm = { workspace = true, optional = true }
|
||||
aes-gcm = { workspace = true, optional = true, features = ["rand_core"] }
|
||||
argon2 = { workspace = true, optional = true }
|
||||
chacha20poly1305 = { workspace = true, optional = true }
|
||||
jsonwebtoken = { workspace = true }
|
||||
jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
|
||||
base64-simd = { workspace = true }
|
||||
pbkdf2 = { workspace = true, optional = true }
|
||||
rand = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
rsa = { workspace = true, features = ["sha2"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
test-case.workspace = true
|
||||
time.workspace = true
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
|
||||
[features]
|
||||
default = ["crypto", "fips"]
|
||||
|
||||
@@ -28,7 +28,7 @@ categories = ["data-structures", "filesystem"]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
path-clean = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
|
||||
+21
-18
@@ -38,44 +38,47 @@ futures.workspace = true
|
||||
rustfs-lock.workspace = true
|
||||
rustfs-protos.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tonic = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
tonic = { workspace = true, features = ["gzip", "deflate"] }
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tokio-stream = { workspace = true }
|
||||
rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
serial_test = { workspace = true }
|
||||
aws-sdk-s3.workspace = true
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-smithy-http-client.workspace = true
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
|
||||
async-trait = { workspace = true }
|
||||
flate2.workspace = true
|
||||
http.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
http-body-util.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "multipart"] }
|
||||
rustfs-signer.workspace = true
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
urlencoding.workspace = true
|
||||
walkdir.workspace = true
|
||||
base64 = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
md5 = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
astral-tokio-tar = { workspace = true }
|
||||
s3s = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
zstd.workspace = true
|
||||
time.workspace = true
|
||||
suppaftp = { workspace = true, features = ["tokio", "rustls-aws-lc-rs"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
suppaftp = { workspace = true, features = ["rustls-aws-lc-rs", "tokio-rustls-aws-lc-rs"] }
|
||||
rcgen.workspace = true
|
||||
local-ip-address.workspace = true
|
||||
anyhow.workspace = true
|
||||
rustls.workspace = true
|
||||
russh = { workspace = true }
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
zip.workspace = true
|
||||
clap.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! E2E coverage for the opt-in per-client S3 API rate limit (backlog#1191):
|
||||
//! the layer must be wired into the real server stack, reject over-limit
|
||||
//! clients with `429` + `Retry-After`, keep health probes exempt, and stay
|
||||
//! completely inert with default configuration.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use serial_test::serial;
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn api_rate_limit_enforces_429_with_retry_after_when_enabled() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
// Burst 50 leaves headroom for the readiness-poll ListBuckets calls that
|
||||
// share the loopback client IP; refill (60 rpm = 1/s) is slow enough that
|
||||
// a rapid burst below reliably exhausts the bucket.
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_API_RATE_LIMIT_ENABLE", "true"),
|
||||
("RUSTFS_API_RATE_LIMIT_RPM", "60"),
|
||||
("RUSTFS_API_RATE_LIMIT_BURST", "50"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let client = local_http_client();
|
||||
let list_buckets_url = format!("{}/", env.url);
|
||||
|
||||
let mut throttled = None;
|
||||
let mut allowed = 0usize;
|
||||
for _ in 0..80 {
|
||||
let response = client.get(&list_buckets_url).send().await?;
|
||||
if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS {
|
||||
throttled = Some(response);
|
||||
break;
|
||||
}
|
||||
allowed += 1;
|
||||
}
|
||||
|
||||
let throttled = throttled.unwrap_or_else(|| panic!("no 429 within 80 rapid requests ({allowed} allowed) at burst 50"));
|
||||
assert!(allowed > 0, "healthy traffic below the burst must not be throttled");
|
||||
info!("rate limit engaged after {allowed} allowed requests");
|
||||
|
||||
let retry_after = throttled
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.expect("429 must carry a numeric Retry-After header");
|
||||
assert!(retry_after >= 1, "Retry-After must be at least one second, got {retry_after}");
|
||||
assert_eq!(
|
||||
throttled.headers().get("x-ratelimit-limit").and_then(|v| v.to_str().ok()),
|
||||
Some("60"),
|
||||
"429 must expose the configured limit"
|
||||
);
|
||||
|
||||
let body = throttled.text().await?;
|
||||
assert!(body.contains("<Code>TooManyRequests</Code>"), "S3-style error body expected: {body}");
|
||||
|
||||
// Health probes stay exempt even while the client budget is exhausted.
|
||||
let health = client.get(format!("{}/health", env.url)).send().await?;
|
||||
assert_ne!(
|
||||
health.status(),
|
||||
reqwest::StatusCode::TOO_MANY_REQUESTS,
|
||||
"health probes must never be rate limited"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn api_rate_limit_stays_inert_by_default() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = local_http_client();
|
||||
let list_buckets_url = format!("{}/", env.url);
|
||||
|
||||
for i in 0..80 {
|
||||
let response = client.get(&list_buckets_url).send().await?;
|
||||
assert_ne!(
|
||||
response.status(),
|
||||
reqwest::StatusCode::TOO_MANY_REQUESTS,
|
||||
"request {i} was throttled although rate limiting is disabled by default"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -446,6 +446,13 @@ impl RustFSTestEnvironment {
|
||||
self.start_rustfs_server_inner(extra_args, &[], false).await
|
||||
}
|
||||
|
||||
pub async fn start_rustfs_server_without_cleanup_with_env(
|
||||
&mut self,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner(vec![], extra_env, false).await
|
||||
}
|
||||
|
||||
/// Wait for RustFS server to be ready.
|
||||
///
|
||||
/// A listening TCP port is not sufficient here: the process may accept
|
||||
|
||||
@@ -67,6 +67,10 @@ mod bucket_policy_check_test;
|
||||
#[cfg(test)]
|
||||
mod security_boundary_test;
|
||||
|
||||
// Opt-in per-client S3 API rate limiting (backlog#1191)
|
||||
#[cfg(test)]
|
||||
mod api_rate_limit_test;
|
||||
|
||||
// Admin authorization gate: non-admin denial + root-credential lifecycle (backlog#1151 sec-4)
|
||||
#[cfg(test)]
|
||||
mod admin_auth_test;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+31
-31
@@ -49,7 +49,7 @@ rustfs-signer.workspace = true
|
||||
rustfs-storage-api.workspace = true
|
||||
rustfs-tls-runtime.workspace = true
|
||||
rustfs-checksums.workspace = true
|
||||
rustfs-config = { workspace = true, features = ["constants", "notify", "audit", "server-config-model"] }
|
||||
rustfs-config = { workspace = true, features = ["notify", "audit", "server-config-model"] }
|
||||
rustfs-concurrency.workspace = true
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-common.workspace = true
|
||||
@@ -62,9 +62,9 @@ rustfs-s3-types = { workspace = true }
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-object-capacity.workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
byteorder = { workspace = true }
|
||||
chrono.workspace = true
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
glob = { workspace = true }
|
||||
thiserror.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
@@ -72,22 +72,22 @@ futures.workspace = true
|
||||
futures-util.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-opentelemetry.workspace = true
|
||||
serde.workspace = true
|
||||
time.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
bytesize.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
|
||||
s3s.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
http.workspace = true
|
||||
opentelemetry.workspace = true
|
||||
http-body = { workspace = true }
|
||||
http-body-util.workspace = true
|
||||
url.workspace = true
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
|
||||
reed-solomon-erasure = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
|
||||
reed-solomon-erasure = { workspace = true, features = ["simd-accel"] }
|
||||
reed-solomon-simd = { workspace = true }
|
||||
lazy_static.workspace = true
|
||||
moka = { workspace = true }
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
rustfs-lock.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
regex = { workspace = true }
|
||||
@@ -101,38 +101,38 @@ sha1 = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
hex-simd = { workspace = true }
|
||||
tempfile.workspace = true
|
||||
hyper.workspace = true
|
||||
hyper-util.workspace = true
|
||||
hyper-rustls.workspace = true
|
||||
rustls.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] }
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-pki-types.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
|
||||
tonic.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "sync", "signal", "fs", "rt-multi-thread"] }
|
||||
tonic = { workspace = true, features = ["gzip", "deflate"] }
|
||||
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
|
||||
tower.workspace = true
|
||||
tower = { workspace = true, features = ["timeout"] }
|
||||
async-channel.workspace = true
|
||||
enumset = { workspace = true }
|
||||
num_cpus = { workspace = true }
|
||||
rand.workspace = true
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
pin-project-lite.workspace = true
|
||||
md-5.workspace = true
|
||||
memmap2 = { workspace = true }
|
||||
libc.workspace = true
|
||||
# "process" adds getrlimit for the io_uring fd-cache RLIMIT_NOFILE gate
|
||||
# (backlog#1178); "fs" comes from the workspace default.
|
||||
rustix = { workspace = true, features = ["process"] }
|
||||
rustix = { workspace = true, features = ["process", "fs"] }
|
||||
rustfs-madmin.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
aes-gcm.workspace = true
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy"] }
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
chacha20poly1305.workspace = true
|
||||
aws-sdk-s3 = { workspace = true }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
urlencoding = { workspace = true }
|
||||
smallvec = { workspace = true }
|
||||
shadow-rs.workspace = true
|
||||
smallvec = { workspace = true, features = ["serde"] }
|
||||
shadow-rs = { workspace = true, default-features = false }
|
||||
async-recursion.workspace = true
|
||||
aws-credential-types = { workspace = true }
|
||||
aws-smithy-types = { workspace = true }
|
||||
aws-smithy-runtime-api = { workspace = true }
|
||||
aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
|
||||
parking_lot = { workspace = true }
|
||||
base64-simd.workspace = true
|
||||
serde_urlencoded.workspace = true
|
||||
@@ -141,7 +141,7 @@ google-cloud-auth = { workspace = true }
|
||||
aws-config = { workspace = true }
|
||||
faster-hex = { workspace = true }
|
||||
ratelimit = { workspace = true }
|
||||
aws-smithy-http-client.workspace = true
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = { workspace = true }
|
||||
@@ -153,19 +153,19 @@ metrics = { workspace = true }
|
||||
rustfs-uring = "0.2.1"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tracing-subscriber = { workspace = true, features = ["json"] }
|
||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||
serial_test = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
proptest = "1"
|
||||
rcgen.workspace = true
|
||||
insta = { workspace = true }
|
||||
insta = { workspace = true, features = ["yaml", "json"] }
|
||||
rustfs-crypto = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
shadow-rs = { workspace = true, features = ["build", "metadata"] }
|
||||
shadow-rs = { workspace = true, default-features = false, features = ["build", "metadata"] }
|
||||
|
||||
[[bench]]
|
||||
name = "erasure_benchmark"
|
||||
|
||||
@@ -260,7 +260,9 @@ pub(crate) fn replication_put_object_header_size(put_options: &PutObjectOptions)
|
||||
|
||||
fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObject<'_> {
|
||||
ReplicationSourceObject {
|
||||
mod_time: object_info.mod_time,
|
||||
mod_time: object_info
|
||||
.mod_time
|
||||
.map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
|
||||
version_id: object_info.version_id.map(|version_id| version_id.to_string()),
|
||||
etag: object_info.etag.as_deref(),
|
||||
actual_size: object_info.get_actual_size().unwrap_or_default(),
|
||||
@@ -463,6 +465,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_action_for_target_head_compares_http_date_precision() {
|
||||
for (source_nanos, target_secs, expected) in [
|
||||
(10_123_456_789, 10, ReplicationAction::None),
|
||||
(-10_876_543_211, -11, ReplicationAction::None),
|
||||
(10_600_000_000, 11, ReplicationAction::All),
|
||||
] {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp_nanos(source_nanos).expect("valid timestamp");
|
||||
let object_info = ObjectInfo {
|
||||
mod_time: Some(mod_time),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
etag: Some("abc123".to_string()),
|
||||
size: 10,
|
||||
..Default::default()
|
||||
};
|
||||
let target = HeadObjectOutput::builder()
|
||||
.last_modified(DateTime::from_secs(target_secs))
|
||||
.version_id(object_info.version_id.expect("version ID").to_string())
|
||||
.e_tag("abc123")
|
||||
.content_length(10)
|
||||
.build();
|
||||
|
||||
assert_eq!(
|
||||
replication_action_for_target_head(&object_info, &target, ReplicationType::Object),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_remove_options_mark_replication_requests() {
|
||||
let mtime = OffsetDateTime::UNIX_EPOCH + Duration::seconds(10);
|
||||
|
||||
@@ -1308,11 +1308,19 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
let object = encode_dir_object(object);
|
||||
let mut opts = opts.clone();
|
||||
let _object_lock_guard = self
|
||||
.acquire_object_write_lock_if_needed("restore_transitioned_object", bucket, &object, &mut opts)
|
||||
.await?;
|
||||
|
||||
if self.single_pool() {
|
||||
return self.pools[0].clone().restore_transitioned_object(bucket, &object, opts).await;
|
||||
return self.pools[0]
|
||||
.clone()
|
||||
.restore_transitioned_object(bucket, &object, &opts)
|
||||
.await;
|
||||
}
|
||||
|
||||
let opts = transition_restore_pool_opts(opts);
|
||||
let opts = transition_restore_pool_opts(&opts);
|
||||
let (_, idx) = self
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
|
||||
.await?;
|
||||
@@ -2171,6 +2179,31 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn restore_transitioned_object_waits_for_existing_reader() {
|
||||
let store = Arc::new(new_read_lock_test_store().await);
|
||||
let ns_lock = store
|
||||
.handle_new_ns_lock("bucket", "object")
|
||||
.await
|
||||
.expect("namespace lock should be created");
|
||||
let _reader = ns_lock
|
||||
.get_read_lock(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("reader should acquire the object lock");
|
||||
|
||||
let err = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
store
|
||||
.clone()
|
||||
.handle_restore_transitioned_object("bucket", "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("restore must wait behind an existing reader")
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(matches!(err, StorageError::Lock(rustfs_lock::LockError::Timeout { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn reader_lock_is_held_when_optimization_is_disabled() {
|
||||
|
||||
@@ -28,11 +28,11 @@ categories = ["web-programming", "development-tools"]
|
||||
doctest = false
|
||||
|
||||
[dependencies]
|
||||
serde.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -34,22 +34,22 @@ hotpath = { workspace = true, optional = true }
|
||||
crc-fast = { workspace = true }
|
||||
rmp.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
serde.workspace = true
|
||||
time.workspace = true
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "sync"] }
|
||||
xxhash-rust = { workspace = true, features = ["xxh64"] }
|
||||
bytes.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "sync", "fs", "rt-multi-thread"] }
|
||||
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
rustfs-utils = { workspace = true, features = ["hash", "http"] }
|
||||
byteorder = { workspace = true }
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
s3s.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
regex.workspace = true
|
||||
arc-swap.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
tempfile = { workspace = true }
|
||||
proptest = "1"
|
||||
|
||||
|
||||
@@ -34,28 +34,27 @@ rustfs-storage-api = { workspace = true }
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-madmin = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync","io-util","time","macros"] }
|
||||
tokio-util = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "io-util", "time", "macros", "fs", "rt-multi-thread"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
tracing = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
thiserror = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "serde"] }
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
async-trait = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
rustfs-test-utils = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
walkdir = { workspace = true }
|
||||
http = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util","fs"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -550,28 +550,37 @@ mod serial_tests {
|
||||
};
|
||||
let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
|
||||
heal_manager.start().await.unwrap();
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec![bucket.to_string()],
|
||||
set_disk_id: SET_DISK_ID.to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
recreate_missing: true,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
timeout: Some(Duration::from_secs(300)),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task_id = request.id.clone();
|
||||
let admission = heal_manager
|
||||
.submit_heal_request(request)
|
||||
.await
|
||||
.expect("failed to submit erasure-set heal");
|
||||
assert!(admission.is_admitted(), "erasure-set heal must be admitted");
|
||||
|
||||
wait_for_task(&heal_manager, &task_id, Duration::from_secs(120)).await;
|
||||
// The erasure-set healer defers any per-version heal that hits a
|
||||
// transient error (unmet quorum, DiskNotFound, a slow-disk read under
|
||||
// load) to a later heal cycle: it persists its resume/checkpoint state
|
||||
// and returns a terminal `Failed { .. "retry scheduled" }` so a *fresh*
|
||||
// heal run re-drives the still-unhealed versions (see the finalize step
|
||||
// in `ErasureSetHealer::heal_bucket_with_resume`). In production the
|
||||
// background scanner is that next run; here we supply it ourselves by
|
||||
// re-submitting the idempotent heal. This keeps the e2e faithful to the
|
||||
// resume design without making it hostage to a rare single-version
|
||||
// transient hiccup — the strict data-restoration assertions below still
|
||||
// run only after a genuine `Completed`.
|
||||
let build_request = |force_start: bool| {
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec![bucket.to_string()],
|
||||
set_disk_id: SET_DISK_ID.to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
recreate_missing: true,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
timeout: Some(Duration::from_secs(300)),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.force_start = force_start;
|
||||
request
|
||||
};
|
||||
drive_heal_to_completion(&heal_manager, build_request, Duration::from_secs(120), 3).await;
|
||||
|
||||
// Every data version is readable end-to-end and physically restored on the
|
||||
// wiped disk. (Resume machinery drove the full per-version heal.)
|
||||
@@ -597,20 +606,74 @@ mod serial_tests {
|
||||
|
||||
/// Poll a heal task to a terminal state, panicking on failure/timeout.
|
||||
async fn wait_for_task(heal_manager: &HealManager, task_id: &str, timeout: Duration) {
|
||||
match await_terminal_status(heal_manager, task_id, timeout).await {
|
||||
HealTaskStatus::Completed => {}
|
||||
HealTaskStatus::Failed { error } => panic!("heal task failed: {error}"),
|
||||
HealTaskStatus::Cancelled => panic!("heal task was cancelled"),
|
||||
other => panic!("heal task reached unexpected terminal state: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll a heal task until it reaches a terminal state and return that state.
|
||||
/// Panics only on timeout — callers decide how to treat each terminal state.
|
||||
async fn await_terminal_status(heal_manager: &HealManager, task_id: &str, timeout: Duration) -> HealTaskStatus {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
if let Ok(status) = heal_manager.get_task_status(task_id).await {
|
||||
match status {
|
||||
HealTaskStatus::Completed => return,
|
||||
HealTaskStatus::Failed { ref error } => panic!("heal task failed: {error}"),
|
||||
HealTaskStatus::Cancelled => panic!("heal task was cancelled"),
|
||||
_ => {}
|
||||
}
|
||||
if let Ok(status) = heal_manager.get_task_status(task_id).await
|
||||
&& matches!(
|
||||
status,
|
||||
HealTaskStatus::Completed
|
||||
| HealTaskStatus::Failed { .. }
|
||||
| HealTaskStatus::Cancelled
|
||||
| HealTaskStatus::Timeout
|
||||
)
|
||||
{
|
||||
return status;
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
panic!("heal task {task_id} did not complete within {timeout:?}");
|
||||
panic!("heal task {task_id} did not reach a terminal state within {timeout:?}");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Submit an erasure-set heal and drive it to a genuine `Completed`,
|
||||
/// tolerating the healer's by-design "retry scheduled" deferral.
|
||||
///
|
||||
/// When a per-version heal hits a transient error the erasure-set healer
|
||||
/// persists its resume state and returns a terminal `Failed` carrying
|
||||
/// `retry scheduled` (retry budget still remaining) — expecting a later heal
|
||||
/// run to finish the job. We re-submit ourselves (an idempotent re-heal),
|
||||
/// mirroring the production background scanner. A `Failed` WITHOUT that
|
||||
/// marker (e.g. `exhausted retries`) or any other non-`Completed` terminal
|
||||
/// state is a real failure and panics.
|
||||
async fn drive_heal_to_completion(
|
||||
heal_manager: &HealManager,
|
||||
build_request: impl Fn(bool) -> HealRequest,
|
||||
per_attempt_timeout: Duration,
|
||||
max_redrives: usize,
|
||||
) {
|
||||
for attempt in 0..=max_redrives {
|
||||
let request = build_request(attempt > 0);
|
||||
let task_id = request.id.clone();
|
||||
let admission = heal_manager
|
||||
.submit_heal_request(request)
|
||||
.await
|
||||
.expect("failed to submit erasure-set heal");
|
||||
assert!(admission.is_admitted(), "erasure-set heal must be admitted");
|
||||
|
||||
match await_terminal_status(heal_manager, &task_id, per_attempt_timeout).await {
|
||||
HealTaskStatus::Completed => return,
|
||||
HealTaskStatus::Failed { error } if error.contains("retry scheduled") => {
|
||||
info!(attempt, error, "erasure-set heal deferred a transient version; re-driving to completion");
|
||||
// Brief settle before the next idempotent re-heal.
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
HealTaskStatus::Failed { error } => panic!("heal task failed (non-retryable): {error}"),
|
||||
HealTaskStatus::Cancelled => panic!("heal task was cancelled"),
|
||||
other => panic!("heal task reached unexpected terminal state: {other:?}"),
|
||||
}
|
||||
}
|
||||
panic!("erasure-set heal did not reach Completed within {} attempt(s)", max_redrives + 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,30 +30,30 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["constants", "server-config-model"] }
|
||||
tokio.workspace = true
|
||||
time = { workspace = true, features = ["serde-human-readable"] }
|
||||
rustfs-config = { workspace = true, features = ["server-config-model"] }
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
time = { workspace = true, features = ["serde-human-readable", "macros"] }
|
||||
serde = { workspace = true, features = ["derive", "rc"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-storage-api = { workspace = true }
|
||||
rustfs-policy.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
async-trait.workspace = true
|
||||
thiserror.workspace = true
|
||||
arc-swap = { workspace = true }
|
||||
rustfs-crypto = { workspace = true }
|
||||
futures.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
jsonwebtoken = { workspace = true }
|
||||
jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
|
||||
tracing.workspace = true
|
||||
rustfs-madmin.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["path"] }
|
||||
rustfs-io-metrics.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
pollster.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
moka = { workspace = true }
|
||||
openidconnect = { workspace = true }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy"] }
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
openidconnect = { workspace = true, default-features = false, features = ["accept-rfc3339-timestamps"] }
|
||||
http = { workspace = true }
|
||||
url = { workspace = true }
|
||||
|
||||
|
||||
@@ -28,15 +28,15 @@ categories = ["development-tools", "filesystem"]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
bytes = { workspace = true }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync"] }
|
||||
tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] }
|
||||
memmap2 = { workspace = true }
|
||||
rustfs-io-metrics = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs"] }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -33,14 +33,14 @@ metrics = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
num_cpus = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync","rt"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
|
||||
tracing = { workspace = true }
|
||||
sysinfo = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
metrics-util = { version = "0.20", features = ["debugging"] }
|
||||
tokio = { workspace = true, features = ["test-util","rt","macros"] }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
+13
-13
@@ -26,32 +26,32 @@ categories = ["authentication", "web-programming"]
|
||||
authors.workspace = true
|
||||
|
||||
[dependencies]
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
reqwest = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tokio = { workspace = true, features = ["rt", "sync"] }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy", "json"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
time = { workspace = true }
|
||||
moka = { workspace = true }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["egress"] }
|
||||
url = { workspace = true }
|
||||
# Middleware dependencies
|
||||
tower = { workspace = true }
|
||||
tower = { workspace = true, features = ["timeout"] }
|
||||
http = { workspace = true }
|
||||
hyper = { workspace = true, features = ["server"] }
|
||||
hyper = { workspace = true, features = ["server", "http2", "http1"] }
|
||||
http-body = { workspace = true }
|
||||
http-body-util = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
futures = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
tower = { workspace = true, features = ["util"] }
|
||||
hyper = { workspace = true, features = ["server"] }
|
||||
serde_json = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "macros", "rt", "time"] }
|
||||
tower = { workspace = true, features = ["util", "timeout"] }
|
||||
hyper = { workspace = true, features = ["server", "http2", "http1"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
temp-env = { workspace = true }
|
||||
|
||||
[[test]]
|
||||
|
||||
@@ -30,19 +30,19 @@ workspace = true
|
||||
[dependencies]
|
||||
# Core dependencies
|
||||
async-trait = { workspace = true }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
uuid = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread", "sync", "time"] }
|
||||
uuid = { workspace = true, features = ["serde", "v4", "fast-rng", "macro-diagnostics"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
# Cryptography
|
||||
aes-gcm = { workspace = true }
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
argon2 = { workspace = true }
|
||||
chacha20poly1305 = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
base64 = { workspace = true }
|
||||
zeroize = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -60,11 +60,11 @@ rustfs-utils = { workspace = true }
|
||||
rustfs-security-governance = { workspace = true }
|
||||
|
||||
# HTTP client for Vault
|
||||
reqwest = { workspace = true }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy"] }
|
||||
vaultrs = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
insta = { workspace = true }
|
||||
insta = { workspace = true, features = ["yaml", "json"] }
|
||||
tempfile = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
|
||||
|
||||
@@ -31,17 +31,17 @@ rustfs-common.workspace = true
|
||||
rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-replication.workspace = true
|
||||
rustfs-storage-api.workspace = true
|
||||
s3s.workspace = true
|
||||
time.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
tracing.workspace = true
|
||||
url.workspace = true
|
||||
uuid = { workspace = true, features = ["v4", "serde"] }
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = "1"
|
||||
serial_test.workspace = true
|
||||
temp-env.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
tokio = { workspace = true, features = ["macros", "fs", "rt-multi-thread"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -33,15 +33,15 @@ rustfs-io-metrics = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
async-trait.workspace = true
|
||||
futures.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
tonic.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tonic = { workspace = true, features = ["gzip", "deflate"] }
|
||||
tracing.workspace = true
|
||||
uuid.workspace = true
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
thiserror.workspace = true
|
||||
parking_lot.workspace = true
|
||||
smallvec.workspace = true
|
||||
smallvec = { workspace = true, features = ["serde"] }
|
||||
smartstring.workspace = true
|
||||
crossbeam-queue = { workspace = true }
|
||||
|
||||
|
||||
@@ -29,13 +29,13 @@ documentation = "https://docs.rs/rustfs-madmin/latest/rustfs_madmin/"
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
chrono.workspace = true
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
humantime.workspace = true
|
||||
hyper.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
sysinfo.workspace = true
|
||||
time.workspace = true
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -26,7 +26,7 @@ categories = ["web-programming", "development-tools", "filesystem"]
|
||||
documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/"
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["notify", "constants", "server-config-model"] }
|
||||
rustfs-config = { workspace = true, features = ["notify", "server-config-model"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
@@ -36,14 +36,14 @@ arc-swap = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
form_urlencoded = { workspace = true }
|
||||
hashbrown = { workspace = true }
|
||||
hashbrown = { workspace = true, features = ["serde", "rayon"] }
|
||||
percent-encoding = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
rustc-hash = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
starshard = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
starshard = { workspace = true, features = ["rayon", "async", "serde"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "fs"] }
|
||||
tracing = { workspace = true }
|
||||
url = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
@@ -56,12 +56,12 @@ metrics = { workspace = true }
|
||||
quick-xml = { workspace = true, features = ["serialize", "serde-types", "encoding"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
axum = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["path"] }
|
||||
serde_json = { workspace = true }
|
||||
criterion = { workspace = true }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -39,14 +39,14 @@ rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-io-metrics = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["os"] }
|
||||
futures = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "time"] }
|
||||
tokio = { workspace = true, features = ["sync", "time", "fs", "rt-multi-thread"] }
|
||||
tracing = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
walkdir = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "rt"] }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }
|
||||
|
||||
@@ -28,22 +28,22 @@ categories = ["web-programming", "development-tools"]
|
||||
doctest = false
|
||||
|
||||
[dependencies]
|
||||
bytes.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
starshard.workspace = true
|
||||
starshard = { workspace = true, features = ["rayon", "async", "serde"] }
|
||||
sysinfo = { workspace = true, features = ["multithread"] }
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["sync", "rt", "time"] }
|
||||
tokio = { workspace = true, features = ["sync", "time", "fs", "rt-multi-thread"] }
|
||||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
metrics-util = { version = "0.20", features = ["debugging"] }
|
||||
# `rt-multi-thread` lets the concurrency stress tests run tasks on real worker
|
||||
# threads, so they exercise true parallelism on the shared singleflight/index
|
||||
# state rather than only cooperative interleaving.
|
||||
tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "time"] }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time", "fs"] }
|
||||
|
||||
[[bench]]
|
||||
name = "cache_bench"
|
||||
|
||||
+13
-14
@@ -60,7 +60,7 @@ workspace = true
|
||||
[dependencies]
|
||||
rustfs-audit = { workspace = true }
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["constants", "observability"] }
|
||||
rustfs-config = { workspace = true, features = ["observability"] }
|
||||
# NOTE: This dependency on rustfs-ecstore is a known architectural limitation.
|
||||
# The obs crate imports types from ecstore for metrics collection.
|
||||
# Breaking this dependency would require defining traits in obs and
|
||||
@@ -75,10 +75,10 @@ rustfs-security-governance = { workspace = true }
|
||||
rustfs-storage-api = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["ip"] }
|
||||
rustix = { workspace = true, features = ["fs", "stdio"] }
|
||||
chrono = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
flate2 = { workspace = true }
|
||||
glob = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
crossbeam-channel = { workspace = true }
|
||||
crossbeam-deque = { workspace = true }
|
||||
@@ -86,21 +86,21 @@ crossbeam-utils = { workspace = true }
|
||||
futures-util = { workspace = true }
|
||||
num_cpus = { workspace = true }
|
||||
opentelemetry = { workspace = true }
|
||||
opentelemetry-appender-tracing = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true }
|
||||
opentelemetry-appender-tracing = { workspace = true, features = ["experimental_span_attributes", "experimental_metadata_attributes"] }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
opentelemetry-stdout = { workspace = true }
|
||||
opentelemetry-otlp = { workspace = true }
|
||||
opentelemetry-semantic-conventions = { workspace = true }
|
||||
opentelemetry-otlp = { workspace = true, features = ["gzip-http", "reqwest-rustls"] }
|
||||
opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
|
||||
percent-encoding = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
tracing = { workspace = true, features = ["std", "attributes"] }
|
||||
tracing-appender = { workspace = true }
|
||||
tracing-error = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
|
||||
tokio-util = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "tracing-log", "local-time", "json"] }
|
||||
tokio = { workspace = true, features = ["sync", "rt-multi-thread", "time", "macros"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
dial9-tokio-telemetry = { workspace = true, optional = true }
|
||||
thiserror = { workspace = true }
|
||||
zstd = { workspace = true, features = ["zstdmt"] }
|
||||
@@ -108,13 +108,12 @@ sysinfo = { workspace = true }
|
||||
nvml-wrapper = { workspace = true, optional = true }
|
||||
|
||||
[target.'cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies]
|
||||
pyroscope = { workspace = true, features = ["backend-pprof-rs"], optional = true }
|
||||
pyroscope = { workspace = true, optional = true, features = ["backend-pprof-rs"] }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = { workspace = true }
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
tempfile = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
|
||||
@@ -30,22 +30,22 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["constants", "opa"] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
rustfs-config = { workspace = true, features = ["opa"] }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] }
|
||||
time = { workspace = true, features = ["serde", "parsing", "formatting", "macros"] }
|
||||
serde = { workspace = true, features = ["derive", "rc"] }
|
||||
serde_json.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
thiserror.workspace = true
|
||||
strum = { workspace = true, features = ["derive"] }
|
||||
rustfs-crypto = { workspace = true }
|
||||
ipnetwork = { workspace = true, features = ["serde"] }
|
||||
base64-simd = { workspace = true }
|
||||
jsonwebtoken = { workspace = true }
|
||||
jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
|
||||
regex = { workspace = true }
|
||||
reqwest.workspace = true
|
||||
chrono.workspace = true
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy", "json"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
tracing.workspace = true
|
||||
moka.workspace = true
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
async-trait.workspace = true
|
||||
futures.workspace = true
|
||||
pollster.workspace = true
|
||||
|
||||
+18
-18
@@ -67,7 +67,7 @@ rustfs-storage-api = { workspace = true }
|
||||
rustfs-tls-runtime = { workspace = true, optional = true }
|
||||
|
||||
# Async dependencies
|
||||
tokio = { workspace = true, features = ["fs", "io-util", "sync", "time"] }
|
||||
tokio = { workspace = true, features = ["fs", "io-util", "net", "rt", "sync", "time"] }
|
||||
tracing = { workspace = true }
|
||||
futures-util = { workspace = true }
|
||||
|
||||
@@ -75,20 +75,20 @@ futures-util = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
# Serialization
|
||||
serde_json = { workspace = true }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
|
||||
# Utilities
|
||||
async-trait = { workspace = true }
|
||||
time = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
|
||||
# S3 API dependencies
|
||||
s3s = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
|
||||
# FTPS specific dependencies (optional)
|
||||
libunftp = { workspace = true, optional = true }
|
||||
libunftp = { workspace = true, optional = true, features = ["experimental"] }
|
||||
unftp-core = { workspace = true, optional = true }
|
||||
rustls = { workspace = true, optional = true }
|
||||
rustls = { workspace = true, default-features = false, optional = true, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
|
||||
# Swift specific dependencies (optional)
|
||||
rustfs-keystone = { workspace = true, optional = true }
|
||||
@@ -96,15 +96,15 @@ rustfs-ecstore = { workspace = true, optional = true }
|
||||
rustfs-rio = { workspace = true, optional = true }
|
||||
axum = { workspace = true, optional = true }
|
||||
http = { workspace = true, optional = true }
|
||||
tower = { workspace = true, optional = true }
|
||||
tower = { workspace = true, optional = true, features = ["timeout"] }
|
||||
regex = { workspace = true, optional = true }
|
||||
percent-encoding = { workspace = true, optional = true }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
uuid = { workspace = true, optional = true }
|
||||
uuid = { workspace = true, optional = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
futures = { workspace = true, optional = true }
|
||||
http-body-util = { workspace = true, optional = true }
|
||||
tokio-util = { workspace = true, optional = true, features = ["rt"] }
|
||||
serde = { workspace = true, optional = true }
|
||||
tokio-util = { workspace = true, optional = true, features = ["rt", "io", "compat"] }
|
||||
serde = { workspace = true, optional = true, features = ["derive"] }
|
||||
urlencoding = { workspace = true, optional = true }
|
||||
md5 = { workspace = true, optional = true }
|
||||
quick-xml = { workspace = true, optional = true, features = ["serialize"] }
|
||||
@@ -117,21 +117,21 @@ async-compression = { workspace = true, optional = true, features = ["tokio", "g
|
||||
|
||||
# WebDAV specific dependencies (optional)
|
||||
dav-server = { workspace = true, optional = true }
|
||||
hyper = { workspace = true, optional = true }
|
||||
hyper-util = { workspace = true, optional = true }
|
||||
tokio-rustls = { workspace = true, optional = true }
|
||||
hyper = { workspace = true, optional = true, features = ["http2", "http1", "server"] }
|
||||
hyper-util = { workspace = true, optional = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
tokio-rustls = { workspace = true, default-features = false, optional = true, features = ["logging", "tls12", "aws-lc-rs"] }
|
||||
|
||||
# SFTP specific dependencies (optional)
|
||||
russh = { workspace = true, optional = true }
|
||||
russh = { workspace = true, optional = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true, optional = true }
|
||||
subtle = { workspace = true, optional = true }
|
||||
socket2 = { workspace = true, optional = true }
|
||||
socket2 = { workspace = true, optional = true, features = ["all"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
proptest = "1"
|
||||
tracing-subscriber = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "rt"] }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "fs"] }
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
all-features = true
|
||||
|
||||
@@ -46,7 +46,6 @@ pub mod formpost;
|
||||
pub mod handler;
|
||||
pub mod object;
|
||||
pub mod quota;
|
||||
pub mod ratelimit;
|
||||
pub mod router;
|
||||
pub mod slo;
|
||||
pub mod staticweb;
|
||||
|
||||
@@ -1,434 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Rate Limiting Support for Swift API
|
||||
//!
|
||||
//! This module implements rate limiting to prevent abuse and ensure fair resource
|
||||
//! allocation across tenants. Rate limits can be applied per-account, per-container,
|
||||
//! or per-IP address.
|
||||
//!
|
||||
//! # Configuration
|
||||
//!
|
||||
//! Rate limits are configured via container metadata:
|
||||
//!
|
||||
//! ```bash
|
||||
//! # Set account-level rate limit: 1000 requests per minute
|
||||
//! swift post -m "X-Account-Meta-Rate-Limit:1000/60"
|
||||
//!
|
||||
//! # Set container-level rate limit: 100 requests per minute
|
||||
//! swift post container -m "X-Container-Meta-Rate-Limit:100/60"
|
||||
//! ```
|
||||
//!
|
||||
//! # Response Headers
|
||||
//!
|
||||
//! Rate limit information is included in all responses:
|
||||
//!
|
||||
//! ```http
|
||||
//! HTTP/1.1 200 OK
|
||||
//! X-RateLimit-Limit: 1000
|
||||
//! X-RateLimit-Remaining: 950
|
||||
//! X-RateLimit-Reset: 1740003600
|
||||
//! ```
|
||||
//!
|
||||
//! When rate limit is exceeded:
|
||||
//!
|
||||
//! ```http
|
||||
//! HTTP/1.1 429 Too Many Requests
|
||||
//! X-RateLimit-Limit: 1000
|
||||
//! X-RateLimit-Remaining: 0
|
||||
//! X-RateLimit-Reset: 1740003600
|
||||
//! Retry-After: 30
|
||||
//! ```
|
||||
//!
|
||||
//! # Algorithm
|
||||
//!
|
||||
//! Uses token bucket algorithm with per-second refill rate:
|
||||
//! - Each request consumes 1 token
|
||||
//! - Tokens refill at configured rate
|
||||
//! - Burst capacity allows temporary spikes
|
||||
|
||||
use super::{SwiftError, SwiftResult};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tracing::debug;
|
||||
|
||||
const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
|
||||
const LOG_SUBSYSTEM_SWIFT_RATELIMIT: &str = "swift_ratelimit";
|
||||
const EVENT_SWIFT_RATELIMIT_STATE: &str = "swift_ratelimit_state";
|
||||
|
||||
/// Rate limit configuration
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RateLimit {
|
||||
/// Maximum requests allowed in time window
|
||||
pub limit: u32,
|
||||
|
||||
/// Time window in seconds
|
||||
pub window_seconds: u32,
|
||||
}
|
||||
|
||||
impl RateLimit {
|
||||
/// Parse rate limit from metadata value
|
||||
///
|
||||
/// Format: "limit/window_seconds" (e.g., "1000/60" = 1000 requests per 60 seconds)
|
||||
pub fn parse(value: &str) -> SwiftResult<Self> {
|
||||
let parts: Vec<&str> = value.split('/').collect();
|
||||
if parts.len() != 2 {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Invalid rate limit format: {}. Expected format: limit/window_seconds",
|
||||
value
|
||||
)));
|
||||
}
|
||||
|
||||
let limit = parts[0]
|
||||
.parse::<u32>()
|
||||
.map_err(|_| SwiftError::BadRequest(format!("Invalid rate limit value: {}", parts[0])))?;
|
||||
|
||||
let window_seconds = parts[1]
|
||||
.parse::<u32>()
|
||||
.map_err(|_| SwiftError::BadRequest(format!("Invalid window value: {}", parts[1])))?;
|
||||
|
||||
if window_seconds == 0 {
|
||||
return Err(SwiftError::BadRequest("Rate limit window cannot be zero".to_string()));
|
||||
}
|
||||
|
||||
Ok(RateLimit { limit, window_seconds })
|
||||
}
|
||||
|
||||
/// Calculate refill rate (tokens per second)
|
||||
pub fn refill_rate(&self) -> f64 {
|
||||
self.limit as f64 / self.window_seconds as f64
|
||||
}
|
||||
}
|
||||
|
||||
/// Token bucket for rate limiting
|
||||
#[derive(Debug, Clone)]
|
||||
struct TokenBucket {
|
||||
/// Maximum tokens (burst capacity)
|
||||
capacity: u32,
|
||||
|
||||
/// Current available tokens
|
||||
tokens: f64,
|
||||
|
||||
/// Refill rate (tokens per second)
|
||||
refill_rate: f64,
|
||||
|
||||
/// Last refill timestamp (Unix seconds)
|
||||
last_refill: u64,
|
||||
}
|
||||
|
||||
impl TokenBucket {
|
||||
fn new(rate_limit: &RateLimit) -> Self {
|
||||
let capacity = rate_limit.limit;
|
||||
let refill_rate = rate_limit.refill_rate();
|
||||
|
||||
TokenBucket {
|
||||
capacity,
|
||||
tokens: capacity as f64, // Start full
|
||||
refill_rate,
|
||||
last_refill: current_timestamp(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to consume a token
|
||||
///
|
||||
/// Returns Ok(remaining_tokens) if successful, Err(retry_after_seconds) if rate limited
|
||||
fn try_consume(&mut self) -> Result<u32, u64> {
|
||||
// Refill tokens based on time elapsed
|
||||
let now = current_timestamp();
|
||||
let elapsed = now.saturating_sub(self.last_refill);
|
||||
|
||||
if elapsed > 0 {
|
||||
let refill_amount = self.refill_rate * elapsed as f64;
|
||||
self.tokens = (self.tokens + refill_amount).min(self.capacity as f64);
|
||||
self.last_refill = now;
|
||||
}
|
||||
|
||||
// Try to consume 1 token
|
||||
if self.tokens >= 1.0 {
|
||||
self.tokens -= 1.0;
|
||||
Ok(self.tokens.floor() as u32)
|
||||
} else {
|
||||
// Calculate retry-after: time until 1 token is available
|
||||
let tokens_needed = 1.0 - self.tokens;
|
||||
let retry_after = (tokens_needed / self.refill_rate).ceil() as u64;
|
||||
Err(retry_after)
|
||||
}
|
||||
}
|
||||
|
||||
/// Get current token count
|
||||
fn remaining(&mut self) -> u32 {
|
||||
// Refill tokens based on time elapsed
|
||||
let now = current_timestamp();
|
||||
let elapsed = now.saturating_sub(self.last_refill);
|
||||
|
||||
if elapsed > 0 {
|
||||
let refill_amount = self.refill_rate * elapsed as f64;
|
||||
self.tokens = (self.tokens + refill_amount).min(self.capacity as f64);
|
||||
self.last_refill = now;
|
||||
}
|
||||
|
||||
self.tokens.floor() as u32
|
||||
}
|
||||
|
||||
/// Get reset timestamp (when bucket will be full)
|
||||
fn reset_timestamp(&self, now: u64) -> u64 {
|
||||
if self.tokens >= self.capacity as f64 {
|
||||
now
|
||||
} else {
|
||||
let tokens_to_refill = self.capacity as f64 - self.tokens;
|
||||
let seconds_to_full = (tokens_to_refill / self.refill_rate).ceil() as u64;
|
||||
now + seconds_to_full
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Global rate limiter state (in-memory)
|
||||
///
|
||||
/// In production, this should be backed by Redis or similar distributed store
|
||||
#[derive(Clone)]
|
||||
pub struct RateLimiter {
|
||||
buckets: Arc<Mutex<HashMap<String, TokenBucket>>>,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
/// Create new rate limiter
|
||||
pub fn new() -> Self {
|
||||
RateLimiter {
|
||||
buckets: Arc::new(Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check and consume rate limit quota
|
||||
///
|
||||
/// Returns (remaining, reset_timestamp) if successful,
|
||||
/// or SwiftError::TooManyRequests if rate limited
|
||||
pub fn check_rate_limit(&self, key: &str, rate_limit: &RateLimit) -> SwiftResult<(u32, u64)> {
|
||||
let mut buckets = self.buckets.lock().expect("operation should succeed");
|
||||
|
||||
// Get or create bucket for this key
|
||||
let bucket = buckets.entry(key.to_string()).or_insert_with(|| TokenBucket::new(rate_limit));
|
||||
|
||||
let now = current_timestamp();
|
||||
let reset = bucket.reset_timestamp(now);
|
||||
|
||||
match bucket.try_consume() {
|
||||
Ok(remaining) => {
|
||||
debug!(event = EVENT_SWIFT_RATELIMIT_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_RATELIMIT, key = %key, remaining, result = "allowed", "swift ratelimit state changed");
|
||||
Ok((remaining, reset))
|
||||
}
|
||||
Err(retry_after) => {
|
||||
debug!(event = EVENT_SWIFT_RATELIMIT_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_RATELIMIT, key = %key, retry_after, result = "limited", "swift ratelimit state changed");
|
||||
Err(SwiftError::TooManyRequests {
|
||||
retry_after,
|
||||
limit: rate_limit.limit,
|
||||
reset,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get current rate limit status without consuming quota
|
||||
pub fn get_status(&self, key: &str, rate_limit: &RateLimit) -> (u32, u64) {
|
||||
let mut buckets = self.buckets.lock().expect("operation should succeed");
|
||||
|
||||
let bucket = buckets.entry(key.to_string()).or_insert_with(|| TokenBucket::new(rate_limit));
|
||||
|
||||
let now = current_timestamp();
|
||||
let remaining = bucket.remaining();
|
||||
let reset = bucket.reset_timestamp(now);
|
||||
|
||||
(remaining, reset)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RateLimiter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Get current Unix timestamp in seconds
|
||||
fn current_timestamp() -> u64 {
|
||||
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()
|
||||
}
|
||||
|
||||
/// Extract rate limit from account or container metadata
|
||||
pub fn extract_rate_limit(metadata: &HashMap<String, String>) -> Option<RateLimit> {
|
||||
// Check for rate limit in metadata
|
||||
if let Some(rate_limit_str) = metadata
|
||||
.get("x-account-meta-rate-limit")
|
||||
.or_else(|| metadata.get("x-container-meta-rate-limit"))
|
||||
{
|
||||
RateLimit::parse(rate_limit_str).ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Build rate limit key for tracking
|
||||
pub fn build_rate_limit_key(account: &str, container: Option<&str>) -> String {
|
||||
if let Some(cont) = container {
|
||||
format!("account:{}:container:{}", account, cont)
|
||||
} else {
|
||||
format!("account:{}", account)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_rate_limit_valid() {
|
||||
let rate_limit = RateLimit::parse("1000/60").expect("operation should succeed");
|
||||
assert_eq!(rate_limit.limit, 1000);
|
||||
assert_eq!(rate_limit.window_seconds, 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_rate_limit_invalid_format() {
|
||||
let result = RateLimit::parse("1000");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_rate_limit_invalid_limit() {
|
||||
let result = RateLimit::parse("not_a_number/60");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_rate_limit_invalid_window() {
|
||||
let result = RateLimit::parse("1000/not_a_number");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_rate_limit_zero_window() {
|
||||
let result = RateLimit::parse("1000/0");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rate_limit_refill_rate() {
|
||||
let rate_limit = RateLimit {
|
||||
limit: 1000,
|
||||
window_seconds: 60,
|
||||
};
|
||||
assert!((rate_limit.refill_rate() - 16.666666).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_token_bucket_consume() {
|
||||
let rate_limit = RateLimit {
|
||||
limit: 10,
|
||||
window_seconds: 60,
|
||||
};
|
||||
let mut bucket = TokenBucket::new(&rate_limit);
|
||||
|
||||
// Should be able to consume up to limit
|
||||
for i in 0..10 {
|
||||
let result = bucket.try_consume();
|
||||
assert!(result.is_ok(), "Token {} should succeed", i);
|
||||
}
|
||||
|
||||
// 11th request should fail
|
||||
let result = bucket.try_consume();
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_token_bucket_remaining() {
|
||||
let rate_limit = RateLimit {
|
||||
limit: 100,
|
||||
window_seconds: 60,
|
||||
};
|
||||
let mut bucket = TokenBucket::new(&rate_limit);
|
||||
|
||||
// Initial: 100 tokens
|
||||
assert_eq!(bucket.remaining(), 100);
|
||||
|
||||
// Consume 10
|
||||
for _ in 0..10 {
|
||||
bucket.try_consume().expect("operation should succeed");
|
||||
}
|
||||
|
||||
assert_eq!(bucket.remaining(), 90);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rate_limiter() {
|
||||
let limiter = RateLimiter::new();
|
||||
let rate_limit = RateLimit {
|
||||
limit: 5,
|
||||
window_seconds: 60,
|
||||
};
|
||||
|
||||
// Should allow 5 requests
|
||||
for _ in 0..5 {
|
||||
let result = limiter.check_rate_limit("test_key", &rate_limit);
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
// 6th request should fail
|
||||
let result = limiter.check_rate_limit("test_key", &rate_limit);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_rate_limit_account() {
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("x-account-meta-rate-limit".to_string(), "1000/60".to_string());
|
||||
|
||||
let rate_limit = extract_rate_limit(&metadata);
|
||||
assert!(rate_limit.is_some());
|
||||
|
||||
let rate_limit = rate_limit.expect("operation should succeed");
|
||||
assert_eq!(rate_limit.limit, 1000);
|
||||
assert_eq!(rate_limit.window_seconds, 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_rate_limit_container() {
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("x-container-meta-rate-limit".to_string(), "100/60".to_string());
|
||||
|
||||
let rate_limit = extract_rate_limit(&metadata);
|
||||
assert!(rate_limit.is_some());
|
||||
|
||||
let rate_limit = rate_limit.expect("operation should succeed");
|
||||
assert_eq!(rate_limit.limit, 100);
|
||||
assert_eq!(rate_limit.window_seconds, 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_rate_limit_none() {
|
||||
let metadata = HashMap::new();
|
||||
let rate_limit = extract_rate_limit(&metadata);
|
||||
assert!(rate_limit.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_rate_limit_key_account() {
|
||||
let key = build_rate_limit_key("AUTH_test", None);
|
||||
assert_eq!(key, "account:AUTH_test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_rate_limit_key_container() {
|
||||
let key = build_rate_limit_key("AUTH_test", Some("my-container"));
|
||||
assert_eq!(key, "account:AUTH_test:container:my-container");
|
||||
}
|
||||
}
|
||||
@@ -43,36 +43,4 @@ mod swift_integration {
|
||||
let parsed = expiration::parse_delete_at(delete_at).unwrap();
|
||||
assert_eq!(parsed, 1740000000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_rate_limit_keys() {
|
||||
let limiter = ratelimit::RateLimiter::new();
|
||||
let rate = ratelimit::RateLimit {
|
||||
limit: 3,
|
||||
window_seconds: 60,
|
||||
};
|
||||
|
||||
// Different keys should have separate limits
|
||||
for _ in 0..3 {
|
||||
assert!(limiter.check_rate_limit("key1", &rate).is_ok());
|
||||
assert!(limiter.check_rate_limit("key2", &rate).is_ok());
|
||||
}
|
||||
|
||||
// Both keys should now be exhausted
|
||||
assert!(limiter.check_rate_limit("key1", &rate).is_err());
|
||||
assert!(limiter.check_rate_limit("key2", &rate).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rate_limit_metadata_extraction() {
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("x-account-meta-rate-limit".to_string(), "1000/60".to_string());
|
||||
|
||||
let rate_limit = ratelimit::extract_rate_limit(&metadata);
|
||||
assert!(rate_limit.is_some());
|
||||
|
||||
let rate_limit = rate_limit.unwrap();
|
||||
assert_eq!(rate_limit.limit, 1000);
|
||||
assert_eq!(rate_limit.window_seconds, 60);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
#![cfg(feature = "swift")]
|
||||
|
||||
use rustfs_protocols::swift::{quota, ratelimit, slo, symlink, sync, tempurl, versioning};
|
||||
use rustfs_protocols::swift::{quota, slo, symlink, sync, tempurl, versioning};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Test sync configuration parsing
|
||||
@@ -92,14 +92,6 @@ fn test_symlink_detection() {
|
||||
let _is_symlink = symlink::is_symlink(&metadata);
|
||||
}
|
||||
|
||||
/// Test rate limit parsing
|
||||
#[test]
|
||||
fn test_rate_limit_parsing() {
|
||||
let rl = ratelimit::RateLimit::parse("100/60").unwrap();
|
||||
assert_eq!(rl.limit, 100);
|
||||
assert_eq!(rl.window_seconds, 60);
|
||||
}
|
||||
|
||||
/// Test quota structure
|
||||
#[test]
|
||||
fn test_quota_structure() {
|
||||
|
||||
@@ -40,10 +40,10 @@ rustfs-tls-runtime.workspace = true
|
||||
rustfs-utils.workspace = true
|
||||
flatbuffers = { workspace = true }
|
||||
prost = { workspace = true }
|
||||
tonic = { workspace = true, features = ["transport", "tls-native-roots", "tls-aws-lc"] }
|
||||
tonic = { workspace = true, features = ["transport", "tls-native-roots", "tls-aws-lc", "gzip", "deflate"] }
|
||||
tonic-prost = { workspace = true }
|
||||
tonic-prost-build = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -26,17 +26,17 @@ categories = ["web-programming", "development-tools", "filesystem"]
|
||||
documentation = "https://docs.rs/rustfs-replication/latest/rustfs_replication/"
|
||||
|
||||
[dependencies]
|
||||
bytes.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
byteorder.workspace = true
|
||||
regex.workspace = true
|
||||
rmp.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
s3s.workspace = true
|
||||
serde.workspace = true
|
||||
time.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
url.workspace = true
|
||||
uuid = { workspace = true, features = ["v4", "serde"] }
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -29,20 +29,20 @@ documentation = "https://docs.rs/rustfs-rio-v2/latest/rustfs_rio_v2/"
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
aes-gcm.workspace = true
|
||||
bytes.workspace = true
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
chacha20poly1305.workspace = true
|
||||
hex.workspace = true
|
||||
hmac.workspace = true
|
||||
minlz.workspace = true
|
||||
pin-project-lite.workspace = true
|
||||
rand.workspace = true
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
rustfs-rio.workspace = true
|
||||
rustfs-utils.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
sha2.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rustfs-filemeta.workspace = true
|
||||
|
||||
+10
-10
@@ -35,32 +35,32 @@ hotpath = ["dep:hotpath", "hotpath/hotpath"]
|
||||
[dependencies]
|
||||
hotpath = { workspace = true, optional = true }
|
||||
arc-swap.workspace = true
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
rand = { workspace = true }
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
http.workspace = true
|
||||
aes-gcm = { workspace = true }
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
crc-fast = { workspace = true }
|
||||
pin-project-lite.workspace = true
|
||||
serde = { workspace = true }
|
||||
bytes.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy", "stream"] }
|
||||
rustls-pki-types.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
faster-hex.workspace = true
|
||||
futures.workspace = true
|
||||
rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-io-metrics.workspace = true
|
||||
rustfs-tls-runtime.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
|
||||
serde_json.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
md-5 = { workspace = true }
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
base64.workspace = true
|
||||
sha1.workspace = true
|
||||
sha2.workspace = true
|
||||
xxhash-rust = { workspace = true }
|
||||
s3s.workspace = true
|
||||
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
hex-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -28,8 +28,8 @@ categories = ["data-structures"]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -28,22 +28,22 @@ documentation = "https://docs.rs/rustfs-s3select-api/latest/rustfs_s3select_api/
|
||||
[dependencies]
|
||||
metrics = { workspace = true }
|
||||
async-trait.workspace = true
|
||||
bytes.workspace = true
|
||||
chrono.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
rustfs-common.workspace = true
|
||||
datafusion = { workspace = true }
|
||||
datafusion = { workspace = true, default-features = false, features = ["parquet", "recursive_protection", "sql"] }
|
||||
rustfs-ecstore.workspace = true
|
||||
rustfs-storage-api.workspace = true
|
||||
futures = { workspace = true }
|
||||
futures-core = { workspace = true }
|
||||
http.workspace = true
|
||||
pin-project-lite.workspace = true
|
||||
s3s.workspace = true
|
||||
serde_json = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
thiserror = { workspace = true }
|
||||
parking_lot.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
tracing.workspace = true
|
||||
transform-stream.workspace = true
|
||||
url.workspace = true
|
||||
|
||||
@@ -29,12 +29,12 @@ documentation = "https://docs.rs/rustfs-s3select-query/latest/rustfs_s3select_qu
|
||||
rustfs-s3select-api = { workspace = true }
|
||||
async-recursion = { workspace = true }
|
||||
async-trait.workspace = true
|
||||
datafusion = { workspace = true }
|
||||
datafusion = { workspace = true, default-features = false, features = ["parquet", "recursive_protection", "sql"] }
|
||||
derive_builder = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
parking_lot = { workspace = true }
|
||||
s3s.workspace = true
|
||||
tokio = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[lib]
|
||||
|
||||
+10
-10
@@ -33,32 +33,32 @@ workspace = true
|
||||
rustfs-config = { workspace = true, features = ["server-config-model"] }
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs","sync","rt","time","macros"] }
|
||||
tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] }
|
||||
tracing = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
thiserror = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
time = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
rmp-serde = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
tokio-util = { workspace = true }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-storage-api = { workspace = true }
|
||||
http = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
s3s = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
metrics = { workspace = true }
|
||||
rustfs-data-usage = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tracing-subscriber = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "serde"] }
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
# Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via
|
||||
# the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6).
|
||||
rustfs-ecstore = { workspace = true, features = ["test-util"] }
|
||||
|
||||
@@ -27,13 +27,13 @@ documentation = "https://docs.rs/rustfs-signer/latest/rustfs_signer/"
|
||||
|
||||
[dependencies]
|
||||
tracing.workspace = true
|
||||
bytes = { workspace = true }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
http.workspace = true
|
||||
time.workspace = true
|
||||
hyper.workspace = true
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
serde_urlencoded.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["full"] }
|
||||
s3s.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
base64-simd.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
|
||||
@@ -32,14 +32,14 @@ async-trait.workspace = true
|
||||
# Storage-facing replication contracts are isolated in src/replication.rs until
|
||||
# the underlying wire types can move without creating a replication/storage-api cycle.
|
||||
rustfs-filemeta.workspace = true
|
||||
serde.workspace = true
|
||||
time.workspace = true
|
||||
uuid.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
[dev-dependencies]
|
||||
insta = { workspace = true }
|
||||
serde_json.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
insta = { workspace = true, features = ["yaml", "json"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
tokio = { workspace = true, features = ["macros", "fs", "rt-multi-thread"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
+19
-19
@@ -12,7 +12,7 @@ categories = ["web-programming", "development-tools", "filesystem"]
|
||||
documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/"
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["notify", "constants", "audit", "server-config-model"] }
|
||||
rustfs-config = { workspace = true, features = ["notify", "audit", "server-config-model"] }
|
||||
rustfs-extension-schema = { workspace = true }
|
||||
rustfs-tls-runtime = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
@@ -20,44 +20,44 @@ rustfs-utils = { workspace = true, features = ["egress"] }
|
||||
async-trait = { workspace = true }
|
||||
async-nats = { workspace = true }
|
||||
deadpool-postgres = { workspace = true }
|
||||
hyper = { workspace = true }
|
||||
hyper-rustls = { workspace = true }
|
||||
lapin = { workspace = true }
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "tls12", "logging", "aws-lc-rs"] }
|
||||
lapin = { workspace = true, default-features = false, features = ["tokio", "rustls", "rustls--aws_lc_rs"] }
|
||||
libc = { workspace = true }
|
||||
pulsar = { workspace = true }
|
||||
pulsar = { workspace = true, default-features = false, features = ["tokio-rustls-runtime", "telemetry"] }
|
||||
regex = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
rumqttc = { workspace = true }
|
||||
redis = { workspace = true }
|
||||
rustls = { workspace = true }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy"] }
|
||||
rumqttc = { workspace = true, features = ["websocket"] }
|
||||
redis = { workspace = true, features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-native-certs = { workspace = true }
|
||||
s3s = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
sha2 = { workspace = true }
|
||||
snap = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
|
||||
tokio-postgres = { workspace = true }
|
||||
tokio-postgres = { workspace = true, default-features = false, features = ["runtime", "with-serde_json-1"] }
|
||||
tokio-postgres-rustls = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
url = { workspace = true }
|
||||
urlencoding = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "v7", "serde"] }
|
||||
uuid = { workspace = true, features = ["v4", "v7", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
sysinfo = { workspace = true, features = ["multithread"] }
|
||||
rustfs-kafka-async = { workspace = true }
|
||||
mysql_async = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
mysql_async = { workspace = true, default-features = false, features = ["default-rustls", "tracing"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
parking_lot = { workspace = true }
|
||||
hashbrown = { workspace = true }
|
||||
hashbrown = { workspace = true, features = ["serde", "rayon"] }
|
||||
arc-swap = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
tempfile = { workspace = true }
|
||||
rcgen = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
|
||||
[[bench]]
|
||||
name = "queue_store_benchmark"
|
||||
|
||||
@@ -28,10 +28,10 @@ documentation = "https://docs.rs/rustfs-test-utils/latest/rustfs_test_utils/"
|
||||
[dependencies]
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-storage-api = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs", "rt"] }
|
||||
tokio-util = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4"] }
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -32,19 +32,19 @@ rustfs-common.workspace = true
|
||||
rustfs-config.workspace = true
|
||||
arc-swap.workspace = true
|
||||
metrics.workspace = true
|
||||
rustls.workspace = true
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-pki-types.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
sha2.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
|
||||
tokio = { workspace = true, features = ["fs", "rt", "sync", "time"] }
|
||||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
rcgen.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
tempfile.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
tokio = { workspace = true, features = ["macros", "rt", "fs"] }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -28,23 +28,23 @@ categories = ["network-programming", "security", "web-programming"]
|
||||
async-trait = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
http = { workspace = true }
|
||||
ipnetwork = { workspace = true }
|
||||
ipnetwork = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
moka = { workspace = true, features = ["future", "sync"] }
|
||||
reqwest = { workspace = true }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "json"] }
|
||||
rustfs-config = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["net"] }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time", "test-util"] }
|
||||
tower = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt", "time"] }
|
||||
tower = { workspace = true, features = ["timeout"] }
|
||||
tracing = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["full", "test-util"] }
|
||||
tower = { workspace = true, features = ["util"] }
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
tower = { workspace = true, features = ["util", "timeout"] }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ categories = ["web-programming", "development-tools", "cryptography"]
|
||||
base64-simd = { workspace = true, optional = true }
|
||||
blake2 = { workspace = true, optional = true }
|
||||
brotli = { workspace = true, optional = true }
|
||||
bytes = { workspace = true, optional = true }
|
||||
bytes = { workspace = true, optional = true, features = ["serde"] }
|
||||
crc-fast = { workspace = true, optional = true }
|
||||
flate2 = { workspace = true, optional = true }
|
||||
futures = { workspace = true, optional = true }
|
||||
@@ -36,21 +36,21 @@ hex-simd = { workspace = true, optional = true }
|
||||
highway = { workspace = true, optional = true }
|
||||
hmac = { workspace = true, optional = true }
|
||||
http = { workspace = true, optional = true }
|
||||
hyper = { workspace = true, optional = true }
|
||||
hyper = { workspace = true, optional = true, features = ["http2", "http1", "server"] }
|
||||
local-ip-address = { workspace = true, optional = true }
|
||||
lz4 = { workspace = true, optional = true }
|
||||
md-5 = { workspace = true, optional = true }
|
||||
netif = { workspace = true, optional = true }
|
||||
regex = { workspace = true, optional = true }
|
||||
rustix = { workspace = true, optional = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
rustix = { workspace = true, optional = true, features = ["fs"] }
|
||||
serde = { workspace = true, optional = true, features = ["derive"] }
|
||||
sha1 = { workspace = true, optional = true }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
convert_case = { workspace = true, optional = true }
|
||||
siphasher = { workspace = true, optional = true }
|
||||
snap = { workspace = true, optional = true }
|
||||
tempfile = { workspace = true, optional = true }
|
||||
tokio = { workspace = true, optional = true, features = ["io-util", "macros"] }
|
||||
tokio = { workspace = true, optional = true, features = ["io-util", "time"] }
|
||||
tracing = { workspace = true }
|
||||
transform-stream = { workspace = true, optional = true }
|
||||
url = { workspace = true, optional = true }
|
||||
@@ -58,12 +58,12 @@ zstd = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
temp-env = { workspace = true }
|
||||
proptest = "1"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows = { workspace = true, optional = true, features = ["Win32_Storage_FileSystem", "Win32_Foundation"] }
|
||||
windows = { workspace = true, optional = true, features = ["Win32_Storage_FileSystem"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -12,9 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::io::Write;
|
||||
use std::io::{self, Write};
|
||||
use std::{fmt, str};
|
||||
use tokio::io;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
|
||||
pub enum CompressionAlgorithm {
|
||||
|
||||
@@ -41,7 +41,7 @@ async-compression = { workspace = true, features = [
|
||||
"zstd",
|
||||
"xz",
|
||||
] }
|
||||
tokio = { workspace = true, features = ["fs","io-util","macros"] }
|
||||
tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread"] }
|
||||
tokio-stream = { workspace = true }
|
||||
astral-tokio-tar = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
@@ -9,22 +9,26 @@
|
||||
> (`.config/nextest.toml`); admission criteria: `crates/e2e_test/README.md`.
|
||||
> 🌙 marks tests in the scheduled `e2e-repl-nightly` profile (backlog#1147
|
||||
> repl-1): `replication_extension_test` splits 20 fast tests into the PR smoke
|
||||
> lane and 23 slow / `_real_dual_node` / `_real_single_node` tests into the
|
||||
> lane and 27 slow / `_real_dual_node` / `_real_three_node` / `_real_single_node` tests into the
|
||||
> nightly lane (`.github/workflows/e2e-replication-nightly.yml`).
|
||||
> Note: counts exclude `#[ignore]`d tests (nextest lists them separately).
|
||||
> The SSE-S3 replication contract is ignored under backlog#1291 until its
|
||||
> plaintext downgrade is fixed.
|
||||
|
||||
| module | tests | PR smoke |
|
||||
|---|---|---|
|
||||
| admin_auth_test | 3 | |
|
||||
| admin_auth_test | 3 | ✅ |
|
||||
| admin_iam_crud_test | 2 | ✅ |
|
||||
| admin_timeout_regression_test | 1 | |
|
||||
| anonymous_access_test | 3 | ✅ |
|
||||
| archive_download_integrity_test | 13 | |
|
||||
| bucket_logging_test | 3 | |
|
||||
| bucket_policy_check_test | 1 | ✅ |
|
||||
| checksum_upload_test | 6 | |
|
||||
| checksum_upload_test | 7 | |
|
||||
| cluster_concurrency_test | 2 | |
|
||||
| common | 3 | |
|
||||
| compression_test | 1 | |
|
||||
| console_smoke_test | 1 | ✅ |
|
||||
| content_encoding_test | 3 | ✅ |
|
||||
| copy_object_metadata_test | 1 | ✅ |
|
||||
| copy_object_version_restore_test | 1 | |
|
||||
@@ -50,7 +54,8 @@
|
||||
| mc_mirror_small_bucket_test | 1 | |
|
||||
| multipart_auth_test | 109 | |
|
||||
| namespace_lock_quorum_test | 2 | |
|
||||
| negative_sigv4_test | 6 | |
|
||||
| negative_sigv4_test | 6 | ✅ |
|
||||
| notification_webhook_test | 1 | ✅ |
|
||||
| object_lambda_test | 16 | |
|
||||
| object_lock | 33 | |
|
||||
| overwrite_cleanup_regression_test | 1 | |
|
||||
@@ -58,14 +63,15 @@
|
||||
| protocols | 16 | |
|
||||
| quota_test | 13 | |
|
||||
| reliability_disk_fault_test | 3 | |
|
||||
| reliant | 9 | 3 ✅ |
|
||||
| replication_extension_test | 43 | 20 ✅ +23 🌙 |
|
||||
| reliant | 10 | 4 ✅ |
|
||||
| replication_extension_test | 47 | 20 ✅ +27 🌙 |
|
||||
| security_boundary_test | 4 | |
|
||||
| server_startup_failfast_test | 1 | |
|
||||
| snowball_auto_extract_test | 6 | |
|
||||
| special_chars_test | 14 | ✅ |
|
||||
| stale_multipart_cleanup_cluster_test | 1 | |
|
||||
| tls_gen | 3 | |
|
||||
| tls_hot_reload_test | 1 | ✅ |
|
||||
| version_id_regression_test | 10 | ✅ |
|
||||
|
||||
**Total listed: 424 tests across 52 modules · PR smoke subset: 93 tests / 20 modules** (18 full modules + 3 `reliant::lifecycle` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 23 tests** · generated 2026-07-15.
|
||||
**Total listed: 435 tests across 56 modules · PR smoke subset: 108 tests / 26 modules** (24 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-15.
|
||||
|
||||
@@ -60,7 +60,7 @@
|
||||
{
|
||||
default = rustPlatform.buildRustPackage {
|
||||
pname = "rustfs";
|
||||
version = "1.0.0-beta.9";
|
||||
version = "1.0.0-beta.10-preview.1";
|
||||
|
||||
src = ./.;
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
||||
name: rustfs
|
||||
description: RustFS helm chart to deploy RustFS on kubernetes cluster.
|
||||
type: application
|
||||
version: "0.9.0"
|
||||
appVersion: "1.0.0-beta.9"
|
||||
version: "0.10.0-preview.1"
|
||||
appVersion: "1.0.0-beta.10-preview.1"
|
||||
home: https://rustfs.com
|
||||
icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg
|
||||
maintainers:
|
||||
|
||||
+8
-4
@@ -1,13 +1,14 @@
|
||||
%global _enable_debug_packages 0
|
||||
%global _empty_manifest_terminate_build 0
|
||||
%global prerelease beta.10-preview.1
|
||||
Name: rustfs
|
||||
Version: 1.0.0
|
||||
Release: beta.9
|
||||
Release: beta.10.preview.1
|
||||
Summary: High-performance distributed object storage for MinIO alternative
|
||||
|
||||
License: Apache-2.0
|
||||
URL: https://github.com/rustfs/rustfs
|
||||
Source0: https://github.com/rustfs/rustfs/archive/refs/tags/%{version}-%{release}.tar.gz
|
||||
Source0: https://github.com/rustfs/rustfs/archive/refs/tags/%{version}-%{prerelease}.tar.gz
|
||||
|
||||
BuildRequires: cargo
|
||||
BuildRequires: rust
|
||||
@@ -26,7 +27,7 @@ BuildRequires: clang-devel
|
||||
RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature, support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation, RustFS provides faster speed and safer distributed features for high-performance object storage.
|
||||
|
||||
%prep
|
||||
%autosetup -n %{name}-%{version}-%{release}
|
||||
%autosetup -n %{name}-%{version}-%{prerelease}
|
||||
|
||||
%build
|
||||
# Set the target directory according to the schema
|
||||
@@ -49,7 +50,7 @@ CARGO_TARGET_DIR=$TARGET_DIR RUSTFLAGS="-C link-arg=-fuse-ld=mold -C link-arg=-l
|
||||
|
||||
%install
|
||||
mkdir -p %buildroot/usr/bin/
|
||||
install %_builddir/%{name}-%{version}-%{release}/target/%_arch/%_arch-unknown-linux-gnu/release/rustfs %buildroot/usr/bin/
|
||||
install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown-linux-gnu/release/rustfs %buildroot/usr/bin/
|
||||
|
||||
%files
|
||||
%license LICENSE
|
||||
@@ -57,6 +58,9 @@ install %_builddir/%{name}-%{version}-%{release}/target/%_arch/%_arch-unknown-li
|
||||
%_bindir/rustfs
|
||||
|
||||
%changelog
|
||||
* Thu Jul 16 2026 houseme <housemecn@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-beta.10-preview.1
|
||||
|
||||
* Tue Jul 14 2026 houseme <housemecn@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-beta.9
|
||||
|
||||
|
||||
+37
-37
@@ -68,7 +68,7 @@ hotpath = { workspace = true, optional = true }
|
||||
rustfs-heal = { workspace = true }
|
||||
rustfs-audit = { workspace = true }
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["constants", "notify", "server-config-model"] }
|
||||
rustfs-config = { workspace = true, features = ["notify", "server-config-model"] }
|
||||
rustfs-crypto = { workspace = true }
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
@@ -104,7 +104,7 @@ rustfs-object-data-cache = { workspace = true }
|
||||
rustfs-concurrency = { workspace = true }
|
||||
rustfs-scanner = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
starshard = { workspace = true }
|
||||
starshard = { workspace = true, features = ["rayon", "async", "serde"] }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-trait = { workspace = true }
|
||||
@@ -112,53 +112,53 @@ axum.workspace = true
|
||||
futures.workspace = true
|
||||
futures-lite.workspace = true
|
||||
futures-util.workspace = true
|
||||
hyper.workspace = true
|
||||
hyper-util.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
http.workspace = true
|
||||
http-body.workspace = true
|
||||
http-body-util.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
socket2 = { workspace = true }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util"] }
|
||||
tokio-rustls = { workspace = true }
|
||||
aws-sdk-s3 = { workspace = true }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream"] }
|
||||
socket2 = { workspace = true, features = ["all"] }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util", "fs"] }
|
||||
tokio-rustls = { workspace = true, default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
tokio-stream.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tonic = { workspace = true }
|
||||
tower.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
tonic = { workspace = true, features = ["gzip", "deflate"] }
|
||||
tower = { workspace = true, features = ["timeout"] }
|
||||
tower-http = { workspace = true, features = ["trace", "compression-full", "cors", "catch-panic", "timeout", "limit", "request-id", "add-extension"] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
flatbuffers.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
quick-xml.workspace = true
|
||||
rustfs-signer.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
serde_urlencoded = { workspace = true }
|
||||
|
||||
# Cryptography and Security
|
||||
rustls = { workspace = true }
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-pki-types = { workspace = true }
|
||||
subtle = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "serde", "macros"] }
|
||||
|
||||
# Utilities and Tools
|
||||
astral-tokio-tar = { workspace = true }
|
||||
atoi = { workspace = true }
|
||||
atomic_enum = { workspace = true }
|
||||
async_zip = { workspace = true }
|
||||
async_zip = { workspace = true, default-features = false, features = ["tokio", "deflate"] }
|
||||
base64 = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
base64-simd.workspace = true
|
||||
clap = { workspace = true }
|
||||
const-str = { workspace = true }
|
||||
datafusion = { workspace = true }
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
const-str = { workspace = true, features = ["std", "proc"] }
|
||||
datafusion = { workspace = true, default-features = false, features = ["parquet", "recursive_protection", "sql"] }
|
||||
hex-simd.workspace = true
|
||||
matchit = { workspace = true }
|
||||
md5.workspace = true
|
||||
@@ -167,18 +167,18 @@ percent-encoding = { workspace = true }
|
||||
pin-project-lite.workspace = true
|
||||
parking_lot = { workspace = true }
|
||||
rust-embed = { workspace = true, features = ["interpolate-folder-path"] }
|
||||
s3s.workspace = true
|
||||
shadow-rs = { workspace = true, features = ["build", "metadata"] }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
shadow-rs = { workspace = true, default-features = false, features = ["build", "metadata"] }
|
||||
sysinfo = { workspace = true, features = ["multithread"] }
|
||||
thiserror = { workspace = true }
|
||||
tracing.workspace = true
|
||||
url = { workspace = true }
|
||||
urlencoding = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
zip = { workspace = true }
|
||||
libc = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
aes-gcm = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
chacha20poly1305 = { workspace = true }
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -186,7 +186,7 @@ metrics = { workspace = true }
|
||||
opentelemetry = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
# Data structures
|
||||
hashbrown = { workspace = true }
|
||||
hashbrown = { workspace = true, features = ["serde", "rayon"] }
|
||||
mimalloc = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
@@ -196,20 +196,19 @@ libsystemd.workspace = true
|
||||
libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
|
||||
|
||||
[dev-dependencies]
|
||||
uuid = { workspace = true, features = ["v4"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
serial_test = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
aws-config = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
insta = { workspace = true }
|
||||
insta = { workspace = true, features = ["yaml", "json"] }
|
||||
proptest = "1"
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tracing-subscriber = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
rsa = { workspace = true }
|
||||
rcgen = { workspace = true }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
# Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via
|
||||
# the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6).
|
||||
rustfs-ecstore = { workspace = true, features = ["test-util"] }
|
||||
@@ -217,11 +216,12 @@ rustfs-ecstore = { workspace = true, features = ["test-util"] }
|
||||
[build-dependencies]
|
||||
http.workspace = true
|
||||
futures.workspace = true
|
||||
s3s = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
hyper-util = { workspace = true, features = [
|
||||
"tokio",
|
||||
"server-auto",
|
||||
"server-graceful",
|
||||
"tracing",
|
||||
] }
|
||||
shadow-rs = { workspace = true, features = ["build"] }
|
||||
shadow-rs = { workspace = true, default-features = false, features = ["build"] }
|
||||
|
||||
@@ -16,6 +16,10 @@ use crate::admin::runtime_sources::{current_oidc_handle, default_admin_usecase};
|
||||
use crate::admin::storage_api::access::RequestContext;
|
||||
use crate::license::has_valid_license;
|
||||
use crate::server::has_path_prefix;
|
||||
use crate::server::rate_limit::{
|
||||
LABEL_RATE_LIMIT_SCOPE, METRIC_HTTP_SERVER_REQUESTS_RATE_LIMITED_TOTAL, RATE_LIMIT_SCOPE_CONSOLE, RateLimitDecision,
|
||||
RateLimitQuota, RateLimiter, apply_throttle_headers, client_ip,
|
||||
};
|
||||
use crate::server::{
|
||||
CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HeaderMapCarrier, HealthProbe, LICENSE, RUSTFS_ADMIN_PREFIX,
|
||||
RequestContextLayer, VERSION, build_health_response_parts, collect_probe_readiness,
|
||||
@@ -36,7 +40,7 @@ use rust_embed::RustEmbed;
|
||||
use serde::Serialize;
|
||||
use std::{
|
||||
net::{IpAddr, SocketAddr},
|
||||
sync::OnceLock,
|
||||
sync::{Arc, OnceLock},
|
||||
time::Duration,
|
||||
};
|
||||
use tower_http::catch_panic::CatchPanicLayer;
|
||||
@@ -569,12 +573,38 @@ fn setup_console_middleware_stack(
|
||||
// Add request body limit (10MB for console uploads)
|
||||
.layer(RequestBodyLimitLayer::new(5 * 1024 * 1024 * 1024));
|
||||
|
||||
// Add rate limiting if enabled
|
||||
// Per-client console rate limiting, sharing the S3 API limiter core
|
||||
// (`server::rate_limit`). Added last so it is the outermost layer and
|
||||
// over-limit requests are rejected before any other middleware runs.
|
||||
// The client identity comes from validated request extensions only
|
||||
// (trusted-proxy `ClientInfo`, then socket peer address); requests
|
||||
// without one fail open rather than trusting spoofable headers.
|
||||
if rate_limit_enable {
|
||||
info!("Console rate limiting enabled: {} requests per minute", rate_limit_rpm);
|
||||
// Note: tower-http doesn't provide a built-in rate limiter, but we have the foundation
|
||||
// For production, you would integrate with a rate limiting service like Redis
|
||||
// For now, we log that it's configured and ready for integration
|
||||
if let Some(quota) = RateLimitQuota::per_minute(rate_limit_rpm, 0) {
|
||||
let limiter = Arc::new(RateLimiter::new(quota));
|
||||
app = app.layer(middleware::from_fn(move |req: Request, next: middleware::Next| {
|
||||
let limiter = limiter.clone();
|
||||
async move {
|
||||
match client_ip(&req).map(|ip| limiter.check(ip)) {
|
||||
Some(RateLimitDecision::Limited(throttle)) => {
|
||||
metrics::counter!(
|
||||
METRIC_HTTP_SERVER_REQUESTS_RATE_LIMITED_TOTAL,
|
||||
LABEL_RATE_LIMIT_SCOPE => RATE_LIMIT_SCOPE_CONSOLE
|
||||
)
|
||||
.increment(1);
|
||||
debug!(retry_after_secs = throttle.retry_after_secs, "Console request rejected by rate limit");
|
||||
let mut response = StatusCode::TOO_MANY_REQUESTS.into_response();
|
||||
apply_throttle_headers(response.headers_mut(), limiter.quota().requests_per_minute, &throttle);
|
||||
response
|
||||
}
|
||||
_ => next.run(req).await,
|
||||
}
|
||||
}
|
||||
}));
|
||||
info!("Console rate limiting enabled: {} requests per minute", rate_limit_rpm);
|
||||
} else {
|
||||
warn!("Console rate limiting enabled but RPM is 0; it stays inactive");
|
||||
}
|
||||
}
|
||||
|
||||
app
|
||||
@@ -864,6 +894,39 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// setup_console_middleware_stack reads ENV_HEALTH_ENDPOINT_ENABLE (see above).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn console_rate_limit_rejects_over_limit_clients_with_429() {
|
||||
let app = setup_console_middleware_stack(parse_cors_origins(None), true, 1, 30);
|
||||
|
||||
let request_for = |last_octet: u8| {
|
||||
let mut request = Request::builder()
|
||||
.uri(format!("{CONSOLE_PREFIX}/index.html"))
|
||||
.body(Body::empty())
|
||||
.expect("failed to build request");
|
||||
request.extensions_mut().insert(crate::server::RemoteAddr(SocketAddr::new(
|
||||
IpAddr::from([198, 51, 100, last_octet]),
|
||||
50000,
|
||||
)));
|
||||
request
|
||||
};
|
||||
|
||||
let first = app.clone().oneshot(request_for(7)).await.expect("request should succeed");
|
||||
assert_ne!(first.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
|
||||
let second = app.clone().oneshot(request_for(7)).await.expect("request should succeed");
|
||||
assert_eq!(second.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
assert!(
|
||||
second.headers().contains_key(http::header::RETRY_AFTER),
|
||||
"console 429 must carry Retry-After"
|
||||
);
|
||||
|
||||
// A different client IP has its own budget.
|
||||
let other = app.oneshot(request_for(8)).await.expect("request should succeed");
|
||||
assert_ne!(other.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn console_trace_layer_records_request_id_on_current_span() {
|
||||
let writer = SharedWriter::default();
|
||||
|
||||
@@ -1673,9 +1673,42 @@ fn storage_credential_from_issued(scope: TableCredentialScope, issued: IssuedTab
|
||||
}
|
||||
}
|
||||
|
||||
fn table_metadata_location_for_client(table_bucket: &str, metadata_location: &str) -> String {
|
||||
if crate::table_catalog::is_reserved_table_object_key(metadata_location) {
|
||||
format!("s3://{table_bucket}/{metadata_location}")
|
||||
} else {
|
||||
metadata_location.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn table_metadata_location_for_catalog(table_bucket: &str, metadata_location: &str) -> S3Result<String> {
|
||||
crate::table_catalog::table_catalog_object_key_from_location(table_bucket, metadata_location)
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "metadata location must be inside the table bucket"))
|
||||
}
|
||||
|
||||
fn table_metadata_for_client(table_bucket: &str, mut metadata: serde_json::Value) -> serde_json::Value {
|
||||
if let Some(metadata_log) = metadata.get_mut("metadata-log").and_then(serde_json::Value::as_array_mut) {
|
||||
for entry in metadata_log {
|
||||
let Some(metadata_file) = entry.get_mut("metadata-file") else {
|
||||
continue;
|
||||
};
|
||||
let Some(metadata_location) = metadata_file.as_str() else {
|
||||
continue;
|
||||
};
|
||||
if crate::table_catalog::is_reserved_table_object_key(metadata_location) {
|
||||
*metadata_file = serde_json::Value::String(table_metadata_location_for_client(table_bucket, metadata_location));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
metadata
|
||||
}
|
||||
|
||||
fn load_table_response_from_entry(entry: crate::table_catalog::TableEntry, metadata: serde_json::Value) -> RestLoadTableResponse {
|
||||
let mut config = BTreeMap::new();
|
||||
let warehouse_location = entry.warehouse_location.clone();
|
||||
let metadata_location = table_metadata_location_for_client(&entry.table_bucket, &entry.metadata_location);
|
||||
let metadata = table_metadata_for_client(&entry.table_bucket, metadata);
|
||||
config.insert("warehouse-location".to_string(), warehouse_location.clone());
|
||||
config.insert(CREDENTIAL_VENDING_CONFIG_KEY.to_string(), CREDENTIAL_VENDING_UNSUPPORTED.to_string());
|
||||
config.insert(
|
||||
@@ -1687,7 +1720,7 @@ fn load_table_response_from_entry(entry: crate::table_catalog::TableEntry, metad
|
||||
config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), CREDENTIAL_MODE_CLIENT_PROVIDED.to_string());
|
||||
|
||||
RestLoadTableResponse {
|
||||
metadata_location: entry.metadata_location,
|
||||
metadata_location,
|
||||
metadata,
|
||||
config,
|
||||
storage_credentials: Vec::new(),
|
||||
@@ -1774,8 +1807,10 @@ fn commit_table_response_from_result(
|
||||
result: crate::table_catalog::TableCommitResult,
|
||||
metadata: serde_json::Value,
|
||||
) -> RestCommitTableResponse {
|
||||
let metadata_location = table_metadata_location_for_client(&result.table.table_bucket, &result.table.metadata_location);
|
||||
let metadata = table_metadata_for_client(&result.table.table_bucket, metadata);
|
||||
RestCommitTableResponse {
|
||||
metadata_location: result.table.metadata_location,
|
||||
metadata_location,
|
||||
metadata,
|
||||
version_token: result.table.version_token,
|
||||
generation: result.table.generation,
|
||||
@@ -1784,8 +1819,9 @@ fn commit_table_response_from_result(
|
||||
}
|
||||
|
||||
fn table_metadata_location_response_from_entry(entry: crate::table_catalog::TableEntry) -> TableMetadataLocationResponse {
|
||||
let metadata_location = table_metadata_location_for_client(&entry.table_bucket, &entry.metadata_location);
|
||||
TableMetadataLocationResponse {
|
||||
metadata_location: entry.metadata_location,
|
||||
metadata_location,
|
||||
version_token: entry.version_token,
|
||||
generation: entry.generation,
|
||||
warehouse_location: entry.warehouse_location,
|
||||
@@ -1798,6 +1834,12 @@ fn table_commit_request_from_rest_request(
|
||||
table: &str,
|
||||
request: RestCommitTableRequest,
|
||||
) -> S3Result<crate::table_catalog::TableCommitRequest> {
|
||||
let expected_metadata_location = request
|
||||
.expected_metadata_location
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires expected-metadata-location"))?;
|
||||
let new_metadata_location = request
|
||||
.new_metadata_location
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires new-metadata-location"))?;
|
||||
Ok(crate::table_catalog::TableCommitRequest {
|
||||
table_bucket: bucket.to_string(),
|
||||
namespace: namespace.public_name(),
|
||||
@@ -1808,12 +1850,8 @@ fn table_commit_request_from_rest_request(
|
||||
expected_version_token: request
|
||||
.expected_version_token
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires expected-version-token"))?,
|
||||
expected_metadata_location: request
|
||||
.expected_metadata_location
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires expected-metadata-location"))?,
|
||||
new_metadata_location: request
|
||||
.new_metadata_location
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires new-metadata-location"))?,
|
||||
expected_metadata_location: table_metadata_location_for_catalog(bucket, &expected_metadata_location)?,
|
||||
new_metadata_location: table_metadata_location_for_catalog(bucket, &new_metadata_location)?,
|
||||
requirements: request.requirements,
|
||||
writer: request.writer,
|
||||
})
|
||||
@@ -1921,7 +1959,8 @@ fn table_entry_from_register_request(
|
||||
}
|
||||
let table = crate::table_catalog::IdentifierSegment::parse(request.name)
|
||||
.map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?;
|
||||
if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table, &request.metadata_location) {
|
||||
let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?;
|
||||
if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table, &metadata_location) {
|
||||
return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory"));
|
||||
}
|
||||
|
||||
@@ -1936,7 +1975,7 @@ fn table_entry_from_register_request(
|
||||
format: "ICEBERG".to_string(),
|
||||
format_version: 2,
|
||||
warehouse_location: format!("s3://{bucket}/tables/{table_id}"),
|
||||
metadata_location: request.metadata_location,
|
||||
metadata_location,
|
||||
version_token: format!("token-{}", Uuid::new_v4()),
|
||||
generation: 1,
|
||||
state: crate::table_catalog::TableCatalogEntryState::Active,
|
||||
@@ -1954,7 +1993,8 @@ fn table_entry_from_import_request(
|
||||
) -> S3Result<crate::table_catalog::TableEntry> {
|
||||
let table = crate::table_catalog::IdentifierSegment::parse(table.to_string())
|
||||
.map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?;
|
||||
if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table, &request.metadata_location) {
|
||||
let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?;
|
||||
if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table, &metadata_location) {
|
||||
return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory"));
|
||||
}
|
||||
|
||||
@@ -1969,7 +2009,7 @@ fn table_entry_from_import_request(
|
||||
format: "ICEBERG".to_string(),
|
||||
format_version: 2,
|
||||
warehouse_location: format!("s3://{bucket}/tables/{table_id}"),
|
||||
metadata_location: request.metadata_location,
|
||||
metadata_location,
|
||||
version_token: format!("token-{}", Uuid::new_v4()),
|
||||
generation: 1,
|
||||
state: crate::table_catalog::TableCatalogEntryState::Active,
|
||||
@@ -3945,12 +3985,13 @@ where
|
||||
};
|
||||
let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string())
|
||||
.map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?;
|
||||
if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &request.metadata_location) {
|
||||
let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?;
|
||||
if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &metadata_location) {
|
||||
return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory"));
|
||||
}
|
||||
let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?;
|
||||
validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?;
|
||||
let target_metadata = read_table_metadata_json(metadata_backend, bucket, &request.metadata_location).await?;
|
||||
let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?;
|
||||
validate_metadata_table_location_in_bucket(bucket, &target_metadata)?;
|
||||
validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?;
|
||||
let commit_request = crate::table_catalog::TableCommitRequest {
|
||||
@@ -3962,7 +4003,7 @@ where
|
||||
operation: "update-metadata-location".to_string(),
|
||||
expected_version_token: request.version_token,
|
||||
expected_metadata_location: current.metadata_location,
|
||||
new_metadata_location: request.metadata_location,
|
||||
new_metadata_location: metadata_location,
|
||||
requirements: Vec::new(),
|
||||
writer: Some("rustfs-metadata-location-api".to_string()),
|
||||
};
|
||||
@@ -4030,7 +4071,8 @@ where
|
||||
let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?;
|
||||
validate_table_commit_requirements(¤t_metadata, &request.requirements)?;
|
||||
let expected_metadata = current_metadata.clone();
|
||||
let next_metadata = apply_table_commit_updates(current_metadata, &request.updates, ¤t.metadata_location)?;
|
||||
let previous_metadata_location = table_metadata_location_for_client(bucket, ¤t.metadata_location);
|
||||
let next_metadata = apply_table_commit_updates(current_metadata, &request.updates, &previous_metadata_location)?;
|
||||
validate_metadata_table_location_in_bucket(bucket, &next_metadata)?;
|
||||
validate_metadata_matches_current_metadata(&expected_metadata, &next_metadata)?;
|
||||
validate_table_snapshot_commit_conflicts(
|
||||
@@ -4242,7 +4284,8 @@ where
|
||||
"action": "remove-snapshots",
|
||||
"snapshot-ids": expired_snapshot_ids.clone()
|
||||
})];
|
||||
let next_metadata = apply_table_commit_updates(current_metadata.clone(), &updates, ¤t.metadata_location)?;
|
||||
let previous_metadata_location = table_metadata_location_for_client(bucket, ¤t.metadata_location);
|
||||
let next_metadata = apply_table_commit_updates(current_metadata.clone(), &updates, &previous_metadata_location)?;
|
||||
validate_metadata_matches_current_metadata(¤t_metadata, &next_metadata)?;
|
||||
validate_metadata_table_location_in_bucket(bucket, &next_metadata)?;
|
||||
let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string())
|
||||
@@ -4611,7 +4654,11 @@ fn external_catalog_bridge_entry_from_request(
|
||||
let external_namespace = validate_external_catalog_field("external namespace", request.external_namespace)?;
|
||||
let external_table = validate_external_catalog_field("external table", request.external_table)?;
|
||||
let external_table_uuid = validate_external_catalog_optional_field("external table uuid", request.external_table_uuid)?;
|
||||
if let Some(metadata_location) = request.metadata_location.as_deref() {
|
||||
let metadata_location = request
|
||||
.metadata_location
|
||||
.map(|metadata_location| table_metadata_location_for_catalog(bucket, &metadata_location))
|
||||
.transpose()?;
|
||||
if let Some(metadata_location) = metadata_location.as_deref() {
|
||||
validate_external_catalog_metadata_location(namespace, table, metadata_location)?;
|
||||
}
|
||||
Ok(crate::table_catalog::ExternalCatalogBridgeEntry {
|
||||
@@ -4624,7 +4671,7 @@ fn external_catalog_bridge_entry_from_request(
|
||||
external_namespace,
|
||||
external_table,
|
||||
external_table_uuid,
|
||||
metadata_location: request.metadata_location,
|
||||
metadata_location,
|
||||
external_version_token: validate_external_catalog_optional_field(
|
||||
"external version token",
|
||||
request.external_version_token,
|
||||
@@ -4723,6 +4770,12 @@ async fn sync_external_catalog_bridge_response<B>(
|
||||
where
|
||||
B: crate::table_catalog::TableCatalogObjectBackend,
|
||||
{
|
||||
let mut request = request;
|
||||
request.metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?;
|
||||
request.expected_metadata_location = request
|
||||
.expected_metadata_location
|
||||
.map(|metadata_location| table_metadata_location_for_catalog(bucket, &metadata_location))
|
||||
.transpose()?;
|
||||
ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?;
|
||||
validate_external_catalog_metadata_location(namespace, table, &request.metadata_location)?;
|
||||
let target_metadata = read_table_metadata_json(metadata_backend, bucket, &request.metadata_location).await?;
|
||||
@@ -4867,12 +4920,13 @@ where
|
||||
};
|
||||
let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string())
|
||||
.map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?;
|
||||
if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &request.metadata_location) {
|
||||
let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?;
|
||||
if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &metadata_location) {
|
||||
return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory"));
|
||||
}
|
||||
let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?;
|
||||
validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?;
|
||||
let target_metadata = read_table_metadata_json(metadata_backend, bucket, &request.metadata_location).await?;
|
||||
let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?;
|
||||
validate_metadata_table_location_in_bucket(bucket, &target_metadata)?;
|
||||
validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?;
|
||||
let commit_request = crate::table_catalog::TableCommitRequest {
|
||||
@@ -4884,7 +4938,7 @@ where
|
||||
operation: "rollback".to_string(),
|
||||
expected_version_token: request.version_token,
|
||||
expected_metadata_location: current.metadata_location,
|
||||
new_metadata_location: request.metadata_location,
|
||||
new_metadata_location: metadata_location,
|
||||
requirements: Vec::new(),
|
||||
writer: Some("rustfs-catalog-rollback-api".to_string()),
|
||||
};
|
||||
@@ -6699,7 +6753,7 @@ mod tests {
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
|
||||
let request: RegisterTableRequest = serde_json::from_value(serde_json::json!({
|
||||
"name": "events",
|
||||
"metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json",
|
||||
"metadata-location": "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json",
|
||||
"overwrite": false
|
||||
}))
|
||||
.expect("request should parse");
|
||||
@@ -6838,7 +6892,7 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
response.metadata_location,
|
||||
".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"
|
||||
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"
|
||||
);
|
||||
assert_eq!(response.metadata["format-version"], 2);
|
||||
assert_eq!(response.metadata["current-schema-id"], 0);
|
||||
@@ -6856,7 +6910,7 @@ mod tests {
|
||||
assert_eq!(response.metadata["table-uuid"], entry.table_uuid);
|
||||
assert!(
|
||||
metadata_backend
|
||||
.object_exists("warehouse", &response.metadata_location)
|
||||
.object_exists("warehouse", &entry.metadata_location)
|
||||
.await
|
||||
.expect("metadata object lookup should succeed")
|
||||
);
|
||||
@@ -6923,7 +6977,8 @@ mod tests {
|
||||
.await
|
||||
.expect("standard commit should succeed");
|
||||
|
||||
let metadata_file_prefix = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-";
|
||||
let metadata_file_prefix =
|
||||
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-";
|
||||
let metadata_file_suffix = ".metadata.json";
|
||||
let generated_commit_id = commit
|
||||
.metadata_location
|
||||
@@ -6938,17 +6993,20 @@ mod tests {
|
||||
assert_eq!(commit.metadata["refs"]["main"]["snapshot-id"], 10);
|
||||
assert_eq!(
|
||||
commit.metadata["metadata-log"][0]["metadata-file"],
|
||||
".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"
|
||||
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"
|
||||
);
|
||||
let committed = store
|
||||
.load_table("warehouse", "analytics", "events")
|
||||
.await
|
||||
.expect("committed table lookup should succeed")
|
||||
.expect("committed table should exist");
|
||||
assert_eq!(committed.metadata_location, commit.metadata_location);
|
||||
assert_eq!(
|
||||
table_metadata_location_for_client("warehouse", &committed.metadata_location),
|
||||
commit.metadata_location
|
||||
);
|
||||
assert!(
|
||||
metadata_backend
|
||||
.object_exists("warehouse", &commit.metadata_location)
|
||||
.object_exists("warehouse", &committed.metadata_location)
|
||||
.await
|
||||
.expect("committed metadata lookup should succeed")
|
||||
);
|
||||
@@ -6981,11 +7039,16 @@ mod tests {
|
||||
assert_eq!(commit.commit_id, commit_id);
|
||||
assert_eq!(
|
||||
commit.metadata_location,
|
||||
".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-11111111-1111-4111-8111-111111111111.metadata.json"
|
||||
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-11111111-1111-4111-8111-111111111111.metadata.json"
|
||||
);
|
||||
let committed = store
|
||||
.load_table("warehouse", "analytics", "events")
|
||||
.await
|
||||
.expect("committed table lookup should succeed")
|
||||
.expect("committed table should exist");
|
||||
assert!(
|
||||
metadata_backend
|
||||
.object_exists("warehouse", &commit.metadata_location)
|
||||
.object_exists("warehouse", &committed.metadata_location)
|
||||
.await
|
||||
.expect("committed metadata lookup should succeed")
|
||||
);
|
||||
@@ -7014,7 +7077,8 @@ mod tests {
|
||||
.await
|
||||
.expect("standard commit should succeed");
|
||||
|
||||
let metadata_file_prefix = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-";
|
||||
let metadata_file_prefix =
|
||||
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-";
|
||||
let metadata_file_suffix = ".metadata.json";
|
||||
let metadata_file_token = commit
|
||||
.metadata_location
|
||||
@@ -7063,7 +7127,7 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
commit.metadata_location,
|
||||
".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-22222222-2222-4222-8222-222222222222.metadata.json"
|
||||
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-22222222-2222-4222-8222-222222222222.metadata.json"
|
||||
);
|
||||
assert_eq!(commit.metadata["properties"]["owner"], "lakehouse");
|
||||
}
|
||||
@@ -7278,7 +7342,7 @@ mod tests {
|
||||
.await
|
||||
.expect("legacy catalog uuid should not block metadata-location update");
|
||||
|
||||
assert_eq!(updated.metadata_location, next_location);
|
||||
assert_eq!(updated.metadata_location, table_metadata_location_for_client("warehouse", next_location));
|
||||
assert_eq!(updated.generation, legacy_entry.generation + 1);
|
||||
}
|
||||
|
||||
@@ -7570,7 +7634,7 @@ mod tests {
|
||||
assert_eq!(snapshot_log.len(), 1);
|
||||
assert_eq!(
|
||||
committed_metadata["metadata-log"][0]["metadata-file"],
|
||||
serde_json::Value::String(current.clone())
|
||||
serde_json::Value::String(table_metadata_location_for_client(bucket, ¤t))
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
@@ -7976,7 +8040,10 @@ mod tests {
|
||||
.expect("external sync should register missing table");
|
||||
|
||||
assert_eq!(synced.action, "registered");
|
||||
assert_eq!(synced.table.metadata_location, metadata_location);
|
||||
assert_eq!(
|
||||
synced.table.metadata_location,
|
||||
table_metadata_location_for_client(bucket, &metadata_location)
|
||||
);
|
||||
let bridge = synced.bridge.bridge.as_ref().expect("bridge state should be returned");
|
||||
assert_eq!(bridge.catalog, "glue");
|
||||
assert_eq!(bridge.last_sync_status.as_deref(), Some("synced"));
|
||||
@@ -8038,10 +8105,10 @@ mod tests {
|
||||
external_namespace: "sales".to_string(),
|
||||
external_table: "orders".to_string(),
|
||||
external_table_uuid: Some("table-uuid".to_string()),
|
||||
metadata_location: next_location.clone(),
|
||||
metadata_location: table_metadata_location_for_client(bucket, &next_location),
|
||||
external_version_token: Some("hms-version-2".to_string()),
|
||||
expected_version_token: Some("token-v1".to_string()),
|
||||
expected_metadata_location: Some(current_location.clone()),
|
||||
expected_metadata_location: Some(table_metadata_location_for_client(bucket, ¤t_location)),
|
||||
commit_id: Some("external-sync-2".to_string()),
|
||||
idempotency_key: Some("external-sync-idempotency-2".to_string()),
|
||||
policy_mode: Some("rustfs-authoritative".to_string()),
|
||||
@@ -8055,7 +8122,7 @@ mod tests {
|
||||
.expect("external sync should commit existing table");
|
||||
|
||||
assert_eq!(synced.action, "committed");
|
||||
assert_eq!(synced.table.metadata_location, next_location);
|
||||
assert_eq!(synced.table.metadata_location, table_metadata_location_for_client(bucket, &next_location));
|
||||
let current = store
|
||||
.load_table(bucket, "analytics", "events")
|
||||
.await
|
||||
@@ -9677,11 +9744,32 @@ mod tests {
|
||||
let metadata = serde_json::json!({
|
||||
"format-version": 2,
|
||||
"table-uuid": "table-uuid",
|
||||
"location": "s3://warehouse/tables/table-id"
|
||||
"location": "s3://warehouse/tables/table-id",
|
||||
"metadata-log": [
|
||||
{
|
||||
"timestamp-ms": 1,
|
||||
"metadata-file": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json"
|
||||
},
|
||||
{
|
||||
"timestamp-ms": 2,
|
||||
"metadata-file": "s3://warehouse/external/metadata/00000.metadata.json"
|
||||
}
|
||||
]
|
||||
});
|
||||
let response = load_table_response_from_entry(table_entry_for_credentials(), metadata.clone());
|
||||
let response = load_table_response_from_entry(table_entry_for_credentials(), metadata);
|
||||
|
||||
assert_eq!(response.metadata, metadata);
|
||||
assert_eq!(
|
||||
response.metadata_location,
|
||||
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"
|
||||
);
|
||||
assert_eq!(
|
||||
response.metadata["metadata-log"][0]["metadata-file"],
|
||||
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json"
|
||||
);
|
||||
assert_eq!(
|
||||
response.metadata["metadata-log"][1]["metadata-file"],
|
||||
"s3://warehouse/external/metadata/00000.metadata.json"
|
||||
);
|
||||
assert!(response.storage_credentials.is_empty());
|
||||
assert_eq!(response.config.get("rustfs.credential-vending"), Some(&"unsupported".to_string()));
|
||||
assert_eq!(
|
||||
@@ -9702,6 +9790,44 @@ mod tests {
|
||||
assert!(!response.config.contains_key("s3.session-token"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_table_response_preserves_format_v4_relative_metadata_log() {
|
||||
let metadata = serde_json::json!({
|
||||
"format-version": 4,
|
||||
"table-uuid": "table-uuid",
|
||||
"metadata-log": [
|
||||
{
|
||||
"timestamp-ms": 1,
|
||||
"metadata-file": "metadata/00000.metadata.json"
|
||||
},
|
||||
{
|
||||
"timestamp-ms": 2,
|
||||
"metadata-file": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json"
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
let response = load_table_response_from_entry(table_entry_for_credentials(), metadata);
|
||||
|
||||
assert_eq!(response.metadata["metadata-log"][0]["metadata-file"], "metadata/00000.metadata.json");
|
||||
assert_eq!(
|
||||
response.metadata["metadata-log"][1]["metadata-file"],
|
||||
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_metadata_location_for_catalog_accepts_only_the_table_bucket() {
|
||||
let object_key = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json";
|
||||
|
||||
assert_eq!(
|
||||
table_metadata_location_for_catalog("warehouse", &format!("s3://warehouse/{object_key}"))
|
||||
.expect("same-bucket metadata location should normalize"),
|
||||
object_key
|
||||
);
|
||||
assert!(table_metadata_location_for_catalog("warehouse", &format!("s3://other/{object_key}")).is_err());
|
||||
}
|
||||
|
||||
fn table_entry_for_credentials() -> crate::table_catalog::TableEntry {
|
||||
crate::table_catalog::TableEntry {
|
||||
version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION,
|
||||
@@ -11031,7 +11157,8 @@ mod tests {
|
||||
.await
|
||||
.expect("table should register");
|
||||
|
||||
assert_eq!(register.metadata_location, metadata_location);
|
||||
let client_metadata_location = table_metadata_location_for_client("warehouse", metadata_location);
|
||||
assert_eq!(register.metadata_location, client_metadata_location);
|
||||
assert_eq!(register.metadata["format-version"], 2);
|
||||
|
||||
let list = list_tables_response(&store, "warehouse", &namespace)
|
||||
@@ -11042,7 +11169,7 @@ mod tests {
|
||||
let load = load_table_response(&store, &metadata_backend, "warehouse", &namespace, "events")
|
||||
.await
|
||||
.expect("table should load");
|
||||
assert_eq!(load.metadata_location, metadata_location);
|
||||
assert_eq!(load.metadata_location, client_metadata_location);
|
||||
assert_eq!(load.metadata["table-uuid"], "table-uuid");
|
||||
|
||||
let current = store
|
||||
@@ -11075,8 +11202,8 @@ mod tests {
|
||||
idempotency_key: Some("retry-1".to_string()),
|
||||
operation: Some("append".to_string()),
|
||||
expected_version_token: Some(current.version_token.clone()),
|
||||
expected_metadata_location: Some(current.metadata_location.clone()),
|
||||
new_metadata_location: Some(next_metadata_location.to_string()),
|
||||
expected_metadata_location: Some(table_metadata_location_for_client("warehouse", ¤t.metadata_location)),
|
||||
new_metadata_location: Some(table_metadata_location_for_client("warehouse", next_metadata_location)),
|
||||
requirements: Vec::new(),
|
||||
updates: Vec::new(),
|
||||
_identifier: None,
|
||||
@@ -11085,7 +11212,10 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("table commit should succeed");
|
||||
assert_eq!(commit.metadata_location, next_metadata_location);
|
||||
assert_eq!(
|
||||
commit.metadata_location,
|
||||
table_metadata_location_for_client("warehouse", next_metadata_location)
|
||||
);
|
||||
assert_eq!(commit.version_token, "token-committed");
|
||||
assert_eq!(commit.generation, current.generation + 1);
|
||||
assert_eq!(commit.commit_id, "commit-1");
|
||||
@@ -11266,6 +11396,10 @@ mod tests {
|
||||
let current = get_table_metadata_location_response(&store, "warehouse", &namespace, "events")
|
||||
.await
|
||||
.expect("metadata location should load");
|
||||
assert_eq!(
|
||||
current.metadata_location,
|
||||
table_metadata_location_for_client("warehouse", current_location)
|
||||
);
|
||||
let next_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json";
|
||||
metadata_backend
|
||||
.put_json(
|
||||
@@ -11286,7 +11420,7 @@ mod tests {
|
||||
&namespace,
|
||||
"events",
|
||||
UpdateTableMetadataLocationRequest {
|
||||
metadata_location: next_location.to_string(),
|
||||
metadata_location: table_metadata_location_for_client("warehouse", next_location),
|
||||
version_token: current.version_token.clone(),
|
||||
commit_id: Some("commit-1".to_string()),
|
||||
idempotency_key: Some("retry-1".to_string()),
|
||||
@@ -11295,7 +11429,7 @@ mod tests {
|
||||
.await
|
||||
.expect("metadata location should update");
|
||||
|
||||
assert_eq!(updated.metadata_location, next_location);
|
||||
assert_eq!(updated.metadata_location, table_metadata_location_for_client("warehouse", next_location));
|
||||
assert_eq!(updated.generation, current.generation + 1);
|
||||
assert_ne!(updated.version_token, current.version_token);
|
||||
}
|
||||
@@ -11371,7 +11505,10 @@ mod tests {
|
||||
let unchanged = get_table_metadata_location_response(&store, "warehouse", &namespace, "events")
|
||||
.await
|
||||
.expect("metadata location should still load");
|
||||
assert_eq!(unchanged.metadata_location, current_location);
|
||||
assert_eq!(
|
||||
unchanged.metadata_location,
|
||||
table_metadata_location_for_client("warehouse", current_location)
|
||||
);
|
||||
assert_eq!(unchanged.generation, current.generation);
|
||||
}
|
||||
|
||||
@@ -11457,7 +11594,10 @@ mod tests {
|
||||
let unchanged = get_table_metadata_location_response(&store, "warehouse", &namespace, "events")
|
||||
.await
|
||||
.expect("metadata location should still load");
|
||||
assert_eq!(unchanged.metadata_location, current_location);
|
||||
assert_eq!(
|
||||
unchanged.metadata_location,
|
||||
table_metadata_location_for_client("warehouse", current_location)
|
||||
);
|
||||
assert_eq!(unchanged.generation, current.generation);
|
||||
}
|
||||
|
||||
@@ -11502,14 +11642,14 @@ mod tests {
|
||||
&namespace,
|
||||
"events",
|
||||
CatalogImportRequest {
|
||||
metadata_location: imported_location.clone(),
|
||||
metadata_location: table_metadata_location_for_client(bucket, &imported_location),
|
||||
properties: BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("catalog import should register table");
|
||||
assert_eq!(imported.metadata_location, imported_location);
|
||||
assert_eq!(imported.metadata_location, table_metadata_location_for_client(bucket, &imported_location));
|
||||
let current = store
|
||||
.load_table(bucket, "analytics", "events")
|
||||
.await
|
||||
@@ -11531,7 +11671,10 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("repeated catalog import should be idempotent");
|
||||
assert_eq!(imported_again.metadata_location, imported_location);
|
||||
assert_eq!(
|
||||
imported_again.metadata_location,
|
||||
table_metadata_location_for_client(bucket, &imported_location)
|
||||
);
|
||||
|
||||
let rollback_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json");
|
||||
backend
|
||||
@@ -11553,7 +11696,7 @@ mod tests {
|
||||
&namespace,
|
||||
"events",
|
||||
RollbackTableRequest {
|
||||
metadata_location: rollback_location.clone(),
|
||||
metadata_location: table_metadata_location_for_client(bucket, &rollback_location),
|
||||
version_token: current.version_token,
|
||||
commit_id: Some("rollback-1".to_string()),
|
||||
idempotency_key: None,
|
||||
@@ -11562,7 +11705,7 @@ mod tests {
|
||||
.await
|
||||
.expect("rollback should commit selected metadata");
|
||||
|
||||
assert_eq!(rollback.metadata_location, rollback_location);
|
||||
assert_eq!(rollback.metadata_location, table_metadata_location_for_client(bucket, &rollback_location));
|
||||
assert_eq!(rollback.commit_id, "rollback-1");
|
||||
}
|
||||
|
||||
|
||||
@@ -1541,7 +1541,11 @@ async fn restore_object_usecase_reports_ongoing_conflict_and_completion() {
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-restore-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "restore/api-object.bin";
|
||||
// Must live under the `test/` prefix: `set_bucket_lifecycle_transition_with_tier`
|
||||
// scopes the transition rule to `<Filter><Prefix>test/</Prefix>`, so an object
|
||||
// outside it never matches, is never enqueued, and never transitions — the
|
||||
// setup `wait_for_transition` would then time out before the restore assertions.
|
||||
let object = "test/restore/api-object.bin";
|
||||
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
@@ -17,6 +17,7 @@ use rustfs_io_metrics::{
|
||||
record_cpu_usage, record_memory_usage, record_process_memory_split,
|
||||
};
|
||||
use serde::Serialize;
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
@@ -213,6 +214,7 @@ fn read_cgroup_memory_snapshot() -> Option<CgroupMemorySnapshot> {
|
||||
read_cgroup_v2().or_else(read_cgroup_v1)
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn numeric_json_value(value: &Value) -> Option<u64> {
|
||||
match value {
|
||||
Value::Number(number) => number
|
||||
@@ -223,6 +225,7 @@ fn numeric_json_value(value: &Value) -> Option<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
|
||||
match value {
|
||||
Value::Object(fields) => fields
|
||||
@@ -234,6 +237,7 @@ fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64> {
|
||||
match value {
|
||||
Value::Object(fields) => {
|
||||
@@ -250,10 +254,12 @@ fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64>
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_current(value: &Value, metric: &str) -> Option<u64> {
|
||||
mimalloc_stat_field(value, metric, "current")
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservation> {
|
||||
let value = serde_json::from_str::<Value>(stats_json).ok()?;
|
||||
let observation = AllocatorMemoryObservation {
|
||||
|
||||
+59
-17
@@ -27,6 +27,7 @@ use crate::server::{
|
||||
RedirectLayer, RequestContextLayer, RequestLoggingLayer, S3ErrorMessageCompatLayer, VirtualHostStyleHintLayer,
|
||||
redact_sensitive_uri_query,
|
||||
},
|
||||
rate_limit::{RateLimitLayer, api_rate_limit_layer_from_env},
|
||||
tls_material::{
|
||||
TlsAcceptorHolder, TlsHandshakeFailureKind, build_acceptor_from_loaded, load_tls_material, spawn_reload_loop,
|
||||
},
|
||||
@@ -111,6 +112,7 @@ const EVENT_HTTP_BIND_FAILED: &str = "http_bind_failed";
|
||||
const EVENT_HTTP_STARTUP_ENDPOINTS: &str = "http_startup_endpoints";
|
||||
const EVENT_HTTP_HOST_ROUTING: &str = "http_host_routing";
|
||||
const EVENT_HTTP_COMPRESSION_STATE: &str = "http_compression_state";
|
||||
const EVENT_API_RATE_LIMIT_STATE: &str = "api_rate_limit_state";
|
||||
const EVENT_HTTP_TRANSPORT_PARAMETERS: &str = "http_transport_parameters";
|
||||
const EVENT_HTTP_ACCEPT_LOOP_STATE: &str = "http_accept_loop_state";
|
||||
const EVENT_HTTP_CONNECTION_DRAIN: &str = "http_connection_drain";
|
||||
@@ -700,6 +702,34 @@ pub async fn start_http_server(
|
||||
);
|
||||
}
|
||||
|
||||
// Per-client S3 API rate limiting (backlog#1191). Built once so every
|
||||
// connection's stack shares the same limiter state; `None` (the default)
|
||||
// leaves the request path unchanged.
|
||||
let api_rate_limit_layer = api_rate_limit_layer_from_env();
|
||||
match &api_rate_limit_layer {
|
||||
Some(layer) => {
|
||||
let quota = layer.quota();
|
||||
info!(
|
||||
event = EVENT_API_RATE_LIMIT_STATE,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_HTTP,
|
||||
state = "enabled",
|
||||
requests_per_minute = quota.requests_per_minute,
|
||||
burst = quota.burst,
|
||||
"API rate limit state changed"
|
||||
);
|
||||
}
|
||||
None => {
|
||||
debug!(
|
||||
event = EVENT_API_RATE_LIMIT_STATE,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_HTTP,
|
||||
state = "disabled",
|
||||
"API rate limit state changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let is_console = config.console_enable;
|
||||
let server_domains_configured = !config.server_domains.is_empty();
|
||||
let task_handle = tokio::spawn(async move {
|
||||
@@ -890,6 +920,7 @@ pub async fn start_http_server(
|
||||
readiness: readiness.clone(),
|
||||
keystone_auth: auth_keystone::get_keystone_auth(),
|
||||
trusted_proxy_layer: rustfs_trusted_proxies::is_enabled().then(|| rustfs_trusted_proxies::layer().clone()),
|
||||
rate_limit_layer: api_rate_limit_layer.clone(),
|
||||
};
|
||||
|
||||
process_connection(socket, tls_acceptor.clone(), connection_ctx, graceful.watcher());
|
||||
@@ -946,6 +977,9 @@ struct ConnectionContext {
|
||||
keystone_auth: Option<Arc<rustfs_keystone::KeystoneAuthProvider>>,
|
||||
/// Pre-computed trusted proxy layer (avoids per-connection is_enabled() check).
|
||||
trusted_proxy_layer: Option<rustfs_trusted_proxies::TrustedProxyLayer>,
|
||||
/// Per-client API rate limit layer; `None` when disabled (the default).
|
||||
/// All clones share one limiter, keeping budgets global across connections.
|
||||
rate_limit_layer: Option<RateLimitLayer>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -1130,6 +1164,7 @@ fn process_connection(
|
||||
readiness,
|
||||
keystone_auth,
|
||||
trusted_proxy_layer,
|
||||
rate_limit_layer,
|
||||
} = context;
|
||||
|
||||
// Build the hybrid service per-connection.
|
||||
@@ -1183,23 +1218,24 @@ fn process_connection(
|
||||
// 5. RequestContextLayer — creates RequestContext in extensions
|
||||
// 6. EmptyBodyContentLengthCompatLayer — adds Content-Length: 0 for known empty-body API routes
|
||||
// 7. CatchPanicLayer — panic → 500
|
||||
// 8. ReadinessGateLayer — blocks until ready
|
||||
// 9. KeystoneAuthLayer — X-Auth-Token validation
|
||||
// 10. TraceLayer — request span creation + metrics
|
||||
// 11. RequestLoggingLayer — single completion event per request
|
||||
// 12. PropagateRequestIdLayer — X-Request-ID → response
|
||||
// 13. CompressionLayer — response compression (whitelist, path-aware)
|
||||
// 14. PathCategoryInjectionLayer — injects path category for compression predicate
|
||||
// 15. S3ErrorMessageCompatLayer — missing S3 error message compatibility
|
||||
// 16. IcebergRestErrorCompatLayer — Iceberg REST JSON error compatibility
|
||||
// 17. ObjectAttributesEtagFixLayer — ETag fix for GetObjectAttributes
|
||||
// 18. ConditionalCorsLayer — S3 API CORS
|
||||
// 19. RedirectLayer — console redirect (conditional)
|
||||
// 20. BodylessStatusFixLayer — clears body for 1xx/204/205/304 responses
|
||||
// 21. HeadRequestBodyFixLayer — strips actual body bytes from HEAD responses
|
||||
// 22. PublicHealthEndpointLayer — handles public health before s3s host parsing
|
||||
// 23. VirtualHostStyleHintLayer — actionable error for unroutable virtual-hosted-style (conditional)
|
||||
// 24. DoubleSlashListBucketsCompatLayer — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
|
||||
// 8. RateLimitLayer — conditional (external stack only), per-client 429 throttling
|
||||
// 9. ReadinessGateLayer — blocks until ready
|
||||
// 10. KeystoneAuthLayer — X-Auth-Token validation
|
||||
// 11. TraceLayer — request span creation + metrics
|
||||
// 12. RequestLoggingLayer — single completion event per request
|
||||
// 13. PropagateRequestIdLayer — X-Request-ID → response
|
||||
// 14. CompressionLayer — response compression (whitelist, path-aware)
|
||||
// 15. PathCategoryInjectionLayer — injects path category for compression predicate
|
||||
// 16. S3ErrorMessageCompatLayer — missing S3 error message compatibility
|
||||
// 17. IcebergRestErrorCompatLayer — Iceberg REST JSON error compatibility
|
||||
// 18. ObjectAttributesEtagFixLayer — ETag fix for GetObjectAttributes
|
||||
// 19. ConditionalCorsLayer — S3 API CORS
|
||||
// 20. RedirectLayer — console redirect (conditional)
|
||||
// 21. BodylessStatusFixLayer — clears body for 1xx/204/205/304 responses
|
||||
// 22. HeadRequestBodyFixLayer — strips actual body bytes from HEAD responses
|
||||
// 23. PublicHealthEndpointLayer — handles public health before s3s host parsing
|
||||
// 24. VirtualHostStyleHintLayer — actionable error for unroutable virtual-hosted-style (conditional)
|
||||
// 25. DoubleSlashListBucketsCompatLayer — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
// Batch 1 intentionally keeps the external and internode stacks behaviorally
|
||||
// identical while giving each path family a named construction boundary.
|
||||
@@ -1223,6 +1259,12 @@ fn process_connection(
|
||||
.layer(InternodeRequestContextLiteLayer)
|
||||
.layer(EmptyBodyContentLengthCompatLayer)
|
||||
.layer(CatchPanicLayer::new())
|
||||
// Per-client API rate limit (backlog#1191): rejects over-limit
|
||||
// requests with 429 before readiness/auth/tracing spend any
|
||||
// work on them, but after the trusted-proxy layer has resolved
|
||||
// a spoof-proof client IP. Absent (None) unless enabled via
|
||||
// RUSTFS_API_RATE_LIMIT_ENABLE with a non-zero RPM.
|
||||
.option_layer(rate_limit_layer.clone())
|
||||
// CRITICAL: Insert ReadinessGateLayer before business logic
|
||||
// This stops requests from hitting IAMAuth or Storage if they are not ready.
|
||||
.layer(ReadinessGateLayer::new(readiness.clone()))
|
||||
|
||||
+700
-107
@@ -12,124 +12,391 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Rate limiting middleware for RustFS
|
||||
//! Per-client request rate limiting for the S3 API (backlog#1191).
|
||||
//!
|
||||
//! This module provides per-client request rate limiting using a token bucket algorithm.
|
||||
//! It helps protect against DoS attacks by limiting the number of requests per client.
|
||||
//! A token-bucket limiter keyed by client IP. Over-limit requests are rejected
|
||||
//! with `429 Too Many Requests`, a `Retry-After` header, and the
|
||||
//! `x-ratelimit-*` headers already used by the Swift protocol error mapping.
|
||||
//!
|
||||
//! Scope and invariants:
|
||||
//! - **Opt-in, default off.** [`api_rate_limit_layer_from_env`] returns `None`
|
||||
//! unless `RUSTFS_API_RATE_LIMIT_ENABLE=true` and `RUSTFS_API_RATE_LIMIT_RPM > 0`;
|
||||
//! the layer is then absent from the service stack and the request path is
|
||||
//! unchanged.
|
||||
//! - **Client identity is never taken from request headers.** The key is the
|
||||
//! validated [`ClientInfo::real_ip`] inserted by the trusted-proxy layer
|
||||
//! (which only honors `X-Forwarded-For` from configured proxies) or, absent
|
||||
//! that, the socket peer address ([`RemoteAddr`]). A raw `X-Forwarded-For`
|
||||
//! header can therefore not be used to escape into an attacker-chosen bucket.
|
||||
//! - **Infra traffic is exempt** ([`is_rate_limit_exempt_path`]): health and
|
||||
//! profiling probes, internode RPC/gRPC, and the console (which has its own
|
||||
//! limiter, sharing this module's [`RateLimiter`] core).
|
||||
//! - This is client-facing abuse protection, not internal backpressure — it is
|
||||
//! unrelated to `workload_admission`, which schedules already-admitted work.
|
||||
//!
|
||||
//! Memory is bounded: buckets live in [`SHARD_COUNT`] independently locked
|
||||
//! shards, each capped and periodically swept. A bucket that has been idle
|
||||
//! long enough to have refilled completely is indistinguishable from a fresh
|
||||
//! one, so eviction never makes the limiter more permissive than a restart.
|
||||
|
||||
use http::{Request, Response};
|
||||
use crate::server::{
|
||||
CONSOLE_PREFIX, FAVICON_PATH, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_HEALTH_CLUSTER_PATH,
|
||||
MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH, MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH,
|
||||
RPC_PREFIX, RemoteAddr, TONIC_PREFIX, has_path_prefix,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::{Either, Ready, ready};
|
||||
use http::{HeaderMap, HeaderValue, Request, Response, StatusCode};
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use metrics::counter;
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::collections::HashMap;
|
||||
use std::hash::{BuildHasher, RandomState};
|
||||
use std::net::IpAddr;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{Duration, Instant};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use tower::{Layer, Service};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
/// Configuration for rate limiting
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RateLimitConfig {
|
||||
/// Maximum number of requests per window
|
||||
pub max_requests: u32,
|
||||
/// Time window duration
|
||||
pub window_duration: Duration,
|
||||
const LOG_COMPONENT_SERVER: &str = "server";
|
||||
const LOG_SUBSYSTEM_RATE_LIMIT: &str = "rate_limit";
|
||||
const EVENT_REQUEST_RATE_LIMITED: &str = "request_rate_limited";
|
||||
|
||||
pub(crate) const METRIC_HTTP_SERVER_REQUESTS_RATE_LIMITED_TOTAL: &str = "rustfs_http_server_requests_rate_limited_total";
|
||||
pub(crate) const LABEL_RATE_LIMIT_SCOPE: &str = "scope";
|
||||
pub(crate) const RATE_LIMIT_SCOPE_S3_API: &str = "s3_api";
|
||||
pub(crate) const RATE_LIMIT_SCOPE_CONSOLE: &str = "console";
|
||||
|
||||
// Header names shared with the Swift error mapping (crates/protocols swift/errors.rs).
|
||||
const X_RATE_LIMIT_LIMIT: &str = "x-ratelimit-limit";
|
||||
const X_RATE_LIMIT_REMAINING: &str = "x-ratelimit-remaining";
|
||||
const X_RATE_LIMIT_RESET: &str = "x-ratelimit-reset";
|
||||
const X_REQUEST_ID: &str = "x-request-id";
|
||||
|
||||
/// Shards bound lock contention: each request locks 1/32 of the key space.
|
||||
const SHARD_COUNT: usize = 32;
|
||||
/// Upper bound on tracked client IPs across all shards (~100 bytes each, so
|
||||
/// worst case a few MiB). Real distinct-IP cardinality is bounded by actual
|
||||
/// TCP peers (or validated proxy clients), never by spoofable headers.
|
||||
const MAX_TRACKED_CLIENTS: usize = 100_000;
|
||||
/// Per-shard sweep cadence for dropping refilled-idle buckets.
|
||||
const CLEANUP_INTERVAL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Sustained-plus-burst request budget for one client.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct RateLimitQuota {
|
||||
/// Sustained budget in requests per minute.
|
||||
pub requests_per_minute: u32,
|
||||
/// Bucket capacity (maximum burst).
|
||||
pub burst: u32,
|
||||
}
|
||||
|
||||
impl Default for RateLimitConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_requests: 100,
|
||||
window_duration: Duration::from_secs(60),
|
||||
impl RateLimitQuota {
|
||||
/// Build a quota from operator configuration. `rpm == 0` means unlimited
|
||||
/// (`None`); `burst == 0` means "same as RPM".
|
||||
pub fn per_minute(rpm: u32, burst: u32) -> Option<Self> {
|
||||
if rpm == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
requests_per_minute: rpm,
|
||||
burst: if burst == 0 { rpm } else { burst },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Token bucket for rate limiting
|
||||
/// Delay hints for a rejected request, in whole seconds (both at least 1).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ThrottleInfo {
|
||||
/// Seconds until at least one token is available again (`Retry-After`).
|
||||
pub retry_after_secs: u64,
|
||||
/// Seconds until the bucket is completely full (`x-ratelimit-reset`).
|
||||
pub reset_after_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RateLimitDecision {
|
||||
Allowed,
|
||||
Limited(ThrottleInfo),
|
||||
}
|
||||
|
||||
/// One client's bucket. Capacity and refill rate are limiter-wide (single
|
||||
/// quota for all clients), so they are not duplicated per entry.
|
||||
#[derive(Debug)]
|
||||
struct TokenBucket {
|
||||
/// Maximum number of tokens
|
||||
max_tokens: u32,
|
||||
/// Current number of tokens
|
||||
tokens: f64,
|
||||
/// Rate at which tokens are refilled (tokens per second)
|
||||
refill_rate: f64,
|
||||
/// Last time tokens were refilled
|
||||
last_refill: Instant,
|
||||
}
|
||||
|
||||
impl TokenBucket {
|
||||
fn new(max_tokens: u32, window_duration: Duration) -> Self {
|
||||
fn full(capacity: f64, now: Instant) -> Self {
|
||||
Self {
|
||||
max_tokens,
|
||||
tokens: max_tokens as f64,
|
||||
refill_rate: max_tokens as f64 / window_duration.as_secs_f64(),
|
||||
last_refill: Instant::now(),
|
||||
tokens: capacity,
|
||||
last_refill: now,
|
||||
}
|
||||
}
|
||||
|
||||
fn try_consume(&mut self) -> bool {
|
||||
self.refill();
|
||||
fn try_consume(&mut self, capacity: f64, refill_per_sec: f64, now: Instant) -> RateLimitDecision {
|
||||
let elapsed = now.saturating_duration_since(self.last_refill).as_secs_f64();
|
||||
self.tokens = (self.tokens + elapsed * refill_per_sec).min(capacity);
|
||||
self.last_refill = now;
|
||||
|
||||
if self.tokens >= 1.0 {
|
||||
self.tokens -= 1.0;
|
||||
true
|
||||
RateLimitDecision::Allowed
|
||||
} else {
|
||||
false
|
||||
let retry_after_secs = (((1.0 - self.tokens) / refill_per_sec).ceil() as u64).max(1);
|
||||
let reset_after_secs = (((capacity - self.tokens) / refill_per_sec).ceil() as u64).max(1);
|
||||
RateLimitDecision::Limited(ThrottleInfo {
|
||||
retry_after_secs,
|
||||
reset_after_secs,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn refill(&mut self) {
|
||||
let now = Instant::now();
|
||||
let elapsed = now.duration_since(self.last_refill).as_secs_f64();
|
||||
self.tokens = (self.tokens + elapsed * self.refill_rate).min(self.max_tokens as f64);
|
||||
self.last_refill = now;
|
||||
/// Idle long enough that refill would have restored a full bucket: the
|
||||
/// entry carries no information a fresh one would not, so it can be
|
||||
/// dropped losslessly.
|
||||
fn is_refilled_idle(&self, capacity: f64, refill_per_sec: f64, now: Instant) -> bool {
|
||||
now.saturating_duration_since(self.last_refill).as_secs_f64() * refill_per_sec >= capacity
|
||||
}
|
||||
}
|
||||
|
||||
/// Rate limiter state
|
||||
#[derive(Debug)]
|
||||
struct RateLimiterState {
|
||||
/// Per-client token buckets
|
||||
struct Shard {
|
||||
clients: HashMap<IpAddr, TokenBucket>,
|
||||
/// Configuration
|
||||
config: RateLimitConfig,
|
||||
next_cleanup: Instant,
|
||||
}
|
||||
|
||||
impl RateLimiterState {
|
||||
fn new(config: RateLimitConfig) -> Self {
|
||||
/// Sharded per-IP token-bucket limiter. Shared by the S3 API tower layer and
|
||||
/// the console middleware (each with its own instance and quota).
|
||||
#[derive(Debug)]
|
||||
pub struct RateLimiter {
|
||||
shards: Vec<Mutex<Shard>>,
|
||||
shard_hasher: RandomState,
|
||||
quota: RateLimitQuota,
|
||||
capacity: f64,
|
||||
refill_per_sec: f64,
|
||||
max_clients_per_shard: usize,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
pub fn new(quota: RateLimitQuota) -> Self {
|
||||
Self::with_limits(quota, SHARD_COUNT, MAX_TRACKED_CLIENTS)
|
||||
}
|
||||
|
||||
fn with_limits(quota: RateLimitQuota, shard_count: usize, max_clients: usize) -> Self {
|
||||
let now = Instant::now();
|
||||
Self {
|
||||
clients: HashMap::new(),
|
||||
config,
|
||||
shards: (0..shard_count.max(1))
|
||||
.map(|_| {
|
||||
Mutex::new(Shard {
|
||||
clients: HashMap::new(),
|
||||
next_cleanup: now + CLEANUP_INTERVAL,
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
shard_hasher: RandomState::new(),
|
||||
quota,
|
||||
capacity: f64::from(quota.burst.max(1)),
|
||||
refill_per_sec: f64::from(quota.requests_per_minute) / 60.0,
|
||||
max_clients_per_shard: (max_clients / shard_count.max(1)).max(1),
|
||||
}
|
||||
}
|
||||
|
||||
fn check_rate_limit(&mut self, client_ip: IpAddr) -> bool {
|
||||
let bucket = self
|
||||
.clients
|
||||
.entry(client_ip)
|
||||
.or_insert_with(|| TokenBucket::new(self.config.max_requests, self.config.window_duration));
|
||||
|
||||
bucket.try_consume()
|
||||
pub fn quota(&self) -> RateLimitQuota {
|
||||
self.quota
|
||||
}
|
||||
|
||||
fn cleanup_expired(&mut self) {
|
||||
let now = Instant::now();
|
||||
let window = self.config.window_duration;
|
||||
self.clients
|
||||
.retain(|_, bucket| now.duration_since(bucket.last_refill) < window * 2);
|
||||
/// Consume one token for `ip`, creating its bucket on first sight.
|
||||
pub fn check(&self, ip: IpAddr) -> RateLimitDecision {
|
||||
self.check_at(ip, Instant::now())
|
||||
}
|
||||
|
||||
fn check_at(&self, ip: IpAddr, now: Instant) -> RateLimitDecision {
|
||||
let shard = &self.shards[self.shard_index(ip)];
|
||||
let mut shard = shard.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
|
||||
if now >= shard.next_cleanup {
|
||||
let (capacity, refill) = (self.capacity, self.refill_per_sec);
|
||||
shard
|
||||
.clients
|
||||
.retain(|_, bucket| !bucket.is_refilled_idle(capacity, refill, now));
|
||||
shard.next_cleanup = now + CLEANUP_INTERVAL;
|
||||
}
|
||||
|
||||
if let Some(bucket) = shard.clients.get_mut(&ip) {
|
||||
return bucket.try_consume(self.capacity, self.refill_per_sec, now);
|
||||
}
|
||||
|
||||
if shard.clients.len() >= self.max_clients_per_shard {
|
||||
// At capacity: reclaim the most idle entry (closest to full, so the
|
||||
// least information is lost). O(shard len) but only under a
|
||||
// distinct-IP flood, where correctness beats micro-latency.
|
||||
if let Some(stalest) = shard
|
||||
.clients
|
||||
.iter()
|
||||
.min_by_key(|(_, bucket)| bucket.last_refill)
|
||||
.map(|(ip, _)| *ip)
|
||||
{
|
||||
shard.clients.remove(&stalest);
|
||||
}
|
||||
}
|
||||
|
||||
shard
|
||||
.clients
|
||||
.entry(ip)
|
||||
.or_insert_with(|| TokenBucket::full(self.capacity, now))
|
||||
.try_consume(self.capacity, self.refill_per_sec, now)
|
||||
}
|
||||
|
||||
fn shard_index(&self, ip: IpAddr) -> usize {
|
||||
(self.shard_hasher.hash_one(ip) as usize) % self.shards.len()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn tracked_clients(&self) -> usize {
|
||||
self.shards
|
||||
.iter()
|
||||
.map(|shard| shard.lock().unwrap_or_else(PoisonError::into_inner).clients.len())
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// Rate limiting layer
|
||||
/// Resolve the client identity for rate limiting.
|
||||
///
|
||||
/// Only trusted sources are consulted: `ClientInfo` is inserted by the
|
||||
/// trusted-proxy layer after validating the proxy chain, and `RemoteAddr` is
|
||||
/// the raw socket peer. Spoofable headers (`X-Forwarded-For`, `X-Real-IP`)
|
||||
/// are deliberately never read here.
|
||||
pub(crate) fn client_ip<B>(req: &Request<B>) -> Option<IpAddr> {
|
||||
if let Some(info) = req.extensions().get::<ClientInfo>() {
|
||||
return Some(info.real_ip);
|
||||
}
|
||||
req.extensions().get::<RemoteAddr>().map(|remote| remote.0.ip())
|
||||
}
|
||||
|
||||
/// Paths exempt from S3 API rate limiting.
|
||||
///
|
||||
/// - Health/profiling probes: kubelet and load-balancer checks must never be
|
||||
/// throttled, or the limiter itself becomes an availability risk.
|
||||
/// - Internode RPC/gRPC: cluster-internal traffic; throttling it would let an
|
||||
/// external flood starve replication and heal.
|
||||
/// - Console: has its own dedicated limiter (`RUSTFS_CONSOLE_RATE_LIMIT_*`).
|
||||
fn is_rate_limit_exempt_path(path: &str) -> bool {
|
||||
matches!(
|
||||
path,
|
||||
HEALTH_PREFIX
|
||||
| HEALTH_COMPAT_LIVE_PATH
|
||||
| HEALTH_READY_PATH
|
||||
| MINIO_HEALTH_LIVE_PATH
|
||||
| MINIO_HEALTH_READY_PATH
|
||||
| MINIO_HEALTH_CLUSTER_PATH
|
||||
| MINIO_HEALTH_CLUSTER_READ_PATH
|
||||
| PROFILE_CPU_PATH
|
||||
| PROFILE_MEMORY_PATH
|
||||
| FAVICON_PATH
|
||||
) || has_path_prefix(path, RPC_PREFIX)
|
||||
|| has_path_prefix(path, TONIC_PREFIX)
|
||||
|| has_path_prefix(path, CONSOLE_PREFIX)
|
||||
}
|
||||
|
||||
/// Apply the standard throttling headers shared by every rate-limited scope.
|
||||
pub(crate) fn apply_throttle_headers(headers: &mut HeaderMap, limit_rpm: u32, throttle: &ThrottleInfo) {
|
||||
let reset_unix = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
.saturating_add(throttle.reset_after_secs);
|
||||
headers.insert(http::header::RETRY_AFTER, u64_header(throttle.retry_after_secs));
|
||||
headers.insert(X_RATE_LIMIT_LIMIT, u64_header(u64::from(limit_rpm)));
|
||||
headers.insert(X_RATE_LIMIT_REMAINING, HeaderValue::from_static("0"));
|
||||
headers.insert(X_RATE_LIMIT_RESET, u64_header(reset_unix));
|
||||
}
|
||||
|
||||
fn u64_header(value: u64) -> HeaderValue {
|
||||
HeaderValue::from_str(&value.to_string()).unwrap_or_else(|_| HeaderValue::from_static("0"))
|
||||
}
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
type BoxBody = http_body_util::combinators::UnsyncBoxBody<Bytes, BoxError>;
|
||||
|
||||
/// Build the S3-style `429` rejection. The request id (already generated by
|
||||
/// `SetRequestIdLayer`, which sits outside this layer) is echoed manually
|
||||
/// because the rejection short-circuits below `PropagateRequestIdLayer`.
|
||||
fn s3_too_many_requests_response(request_id: Option<&HeaderValue>, limit_rpm: u32, throttle: &ThrottleInfo) -> Response<BoxBody> {
|
||||
// The header may be client-supplied (SetRequestIdLayer only fills it when
|
||||
// absent), so gate the XML interpolation on a UUID-safe charset instead of
|
||||
// reflecting arbitrary bytes into the body.
|
||||
let request_id_xml = request_id
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.filter(|id| !id.is_empty() && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'))
|
||||
.map(|id| format!("<RequestId>{id}</RequestId>"))
|
||||
.unwrap_or_default();
|
||||
let body = format!(
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||
<Error><Code>TooManyRequests</Code>\
|
||||
<Message>Request rate limit exceeded. Reduce your request rate and retry after the indicated delay.</Message>\
|
||||
{request_id_xml}</Error>"
|
||||
);
|
||||
let body: BoxBody = Full::new(Bytes::from(body))
|
||||
.map_err(|e| -> BoxError { Box::new(e) })
|
||||
.boxed_unsync();
|
||||
|
||||
let mut response = Response::new(body);
|
||||
*response.status_mut() = StatusCode::TOO_MANY_REQUESTS;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/xml"));
|
||||
apply_throttle_headers(response.headers_mut(), limit_rpm, throttle);
|
||||
if let Some(id) = request_id {
|
||||
response.headers_mut().insert(X_REQUEST_ID, id.clone());
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// Build the S3 API rate limit layer from `RUSTFS_API_RATE_LIMIT_*`.
|
||||
///
|
||||
/// Returns `None` (no layer in the stack, zero request-path change) unless
|
||||
/// explicitly enabled with a non-zero RPM.
|
||||
pub fn api_rate_limit_layer_from_env() -> Option<RateLimitLayer> {
|
||||
if !rustfs_utils::get_env_bool(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, rustfs_config::DEFAULT_API_RATE_LIMIT_ENABLE) {
|
||||
return None;
|
||||
}
|
||||
let rpm = rustfs_utils::get_env_u32(rustfs_config::ENV_API_RATE_LIMIT_RPM, rustfs_config::DEFAULT_API_RATE_LIMIT_RPM);
|
||||
let burst = rustfs_utils::get_env_u32(rustfs_config::ENV_API_RATE_LIMIT_BURST, rustfs_config::DEFAULT_API_RATE_LIMIT_BURST);
|
||||
let Some(quota) = RateLimitQuota::per_minute(rpm, burst) else {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_RATE_LIMIT,
|
||||
"{} is enabled but {} is 0; API rate limiting stays inactive",
|
||||
rustfs_config::ENV_API_RATE_LIMIT_ENABLE,
|
||||
rustfs_config::ENV_API_RATE_LIMIT_RPM
|
||||
);
|
||||
return None;
|
||||
};
|
||||
Some(RateLimitLayer::new(quota))
|
||||
}
|
||||
|
||||
/// Tower layer enforcing the per-IP quota on the external S3 service stack.
|
||||
///
|
||||
/// All clones share one [`RateLimiter`], so per-connection stack construction
|
||||
/// keeps a single global view of client budgets.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RateLimitLayer {
|
||||
state: Arc<RwLock<RateLimiterState>>,
|
||||
limiter: Arc<RateLimiter>,
|
||||
}
|
||||
|
||||
impl RateLimitLayer {
|
||||
pub fn new(config: RateLimitConfig) -> Self {
|
||||
pub fn new(quota: RateLimitQuota) -> Self {
|
||||
Self {
|
||||
state: Arc::new(RwLock::new(RateLimiterState::new(config))),
|
||||
limiter: Arc::new(RateLimiter::new(quota)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn quota(&self) -> RateLimitQuota {
|
||||
self.limiter.quota()
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for RateLimitLayer {
|
||||
@@ -138,75 +405,401 @@ impl<S> Layer<S> for RateLimitLayer {
|
||||
fn layer(&self, inner: S) -> Self::Service {
|
||||
RateLimitService {
|
||||
inner,
|
||||
state: self.state.clone(),
|
||||
limiter: self.limiter.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Rate limiting service
|
||||
/// See [`RateLimitLayer`]. The response type is pinned to the stack's
|
||||
/// `BoxBody` so the layer composes with `option_layer` at its position just
|
||||
/// outside `ReadinessGateLayer` (which already boxes response bodies).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RateLimitService<S> {
|
||||
inner: S,
|
||||
state: Arc<RwLock<RateLimiterState>>,
|
||||
limiter: Arc<RateLimiter>,
|
||||
}
|
||||
|
||||
impl<S, ReqBody, ResBody> Service<Request<ReqBody>> for RateLimitService<S>
|
||||
impl<S, ReqBody> Service<Request<ReqBody>> for RateLimitService<S>
|
||||
where
|
||||
S: Service<Request<ReqBody>, Response = Response<ResBody>>,
|
||||
S: Service<Request<ReqBody>, Response = Response<BoxBody>>,
|
||||
{
|
||||
type Response = S::Response;
|
||||
type Response = Response<BoxBody>;
|
||||
type Error = S::Error;
|
||||
type Future = S::Future;
|
||||
type Future = Either<S::Future, Ready<Result<Response<BoxBody>, S::Error>>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> std::task::Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, req: Request<ReqBody>) -> Self::Future {
|
||||
// Extract client IP from request
|
||||
let client_ip = extract_client_ip(&req);
|
||||
|
||||
// Check rate limit
|
||||
let allowed = {
|
||||
let mut state = self.state.write().unwrap_or_else(|e| e.into_inner());
|
||||
// Periodically cleanup expired entries
|
||||
if rand::random::<u32>().is_multiple_of(100) {
|
||||
state.cleanup_expired();
|
||||
}
|
||||
state.check_rate_limit(client_ip)
|
||||
if is_rate_limit_exempt_path(req.uri().path()) {
|
||||
return Either::Left(self.inner.call(req));
|
||||
}
|
||||
let Some(ip) = client_ip(&req) else {
|
||||
// Fail open: without a socket peer address there is no trustworthy
|
||||
// client identity, and deriving one from spoofable headers would
|
||||
// let clients pick their own bucket.
|
||||
return Either::Left(self.inner.call(req));
|
||||
};
|
||||
|
||||
if allowed {
|
||||
self.inner.call(req)
|
||||
} else {
|
||||
// Return 429 Too Many Requests
|
||||
// Note: In a real implementation, you'd want to return a proper 429 response.
|
||||
// For now, we just pass the request through to the inner service.
|
||||
// The rate limiting is enforced at the application level.
|
||||
self.inner.call(req)
|
||||
match self.limiter.check(ip) {
|
||||
RateLimitDecision::Allowed => Either::Left(self.inner.call(req)),
|
||||
RateLimitDecision::Limited(throttle) => {
|
||||
counter!(
|
||||
METRIC_HTTP_SERVER_REQUESTS_RATE_LIMITED_TOTAL,
|
||||
LABEL_RATE_LIMIT_SCOPE => RATE_LIMIT_SCOPE_S3_API
|
||||
)
|
||||
.increment(1);
|
||||
debug!(
|
||||
event = EVENT_REQUEST_RATE_LIMITED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_RATE_LIMIT,
|
||||
scope = RATE_LIMIT_SCOPE_S3_API,
|
||||
client_ip = %ip,
|
||||
retry_after_secs = throttle.retry_after_secs,
|
||||
"Request rejected by API rate limit"
|
||||
);
|
||||
let limit_rpm = self.limiter.quota().requests_per_minute;
|
||||
Either::Right(ready(Ok(s3_too_many_requests_response(
|
||||
req.headers().get(X_REQUEST_ID),
|
||||
limit_rpm,
|
||||
&throttle,
|
||||
))))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract client IP from request headers or connection info
|
||||
fn extract_client_ip<ReqBody>(req: &Request<ReqBody>) -> IpAddr {
|
||||
// Try X-Forwarded-For header first
|
||||
if let Some(forwarded) = req.headers().get("x-forwarded-for")
|
||||
&& let Ok(forwarded_str) = forwarded.to_str()
|
||||
&& let Some(first_ip) = forwarded_str.split(',').next()
|
||||
&& let Ok(ip) = first_ip.trim().parse::<IpAddr>()
|
||||
{
|
||||
return ip;
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http_body_util::BodyExt;
|
||||
use serial_test::serial;
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
fn quota(rpm: u32, burst: u32) -> RateLimitQuota {
|
||||
RateLimitQuota::per_minute(rpm, burst).expect("non-zero rpm")
|
||||
}
|
||||
|
||||
// Try X-Real-IP header
|
||||
if let Some(real_ip) = req.headers().get("x-real-ip")
|
||||
&& let Ok(ip_str) = real_ip.to_str()
|
||||
&& let Ok(ip) = ip_str.parse::<IpAddr>()
|
||||
{
|
||||
return ip;
|
||||
fn ip(last: u8) -> IpAddr {
|
||||
IpAddr::V4(Ipv4Addr::new(203, 0, 113, last))
|
||||
}
|
||||
|
||||
// Default to localhost
|
||||
IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)
|
||||
#[test]
|
||||
fn quota_zero_rpm_means_unlimited_and_zero_burst_defaults_to_rpm() {
|
||||
assert_eq!(RateLimitQuota::per_minute(0, 50), None);
|
||||
assert_eq!(quota(120, 0).burst, 120);
|
||||
assert_eq!(quota(120, 10).burst, 10);
|
||||
assert_eq!(quota(120, 500).burst, 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_exhaustion_returns_429_hints_and_window_recovers() {
|
||||
let limiter = RateLimiter::new(quota(60, 5)); // 1 token/sec, burst 5
|
||||
let start = Instant::now();
|
||||
|
||||
for _ in 0..5 {
|
||||
assert_eq!(limiter.check_at(ip(1), start), RateLimitDecision::Allowed);
|
||||
}
|
||||
let RateLimitDecision::Limited(throttle) = limiter.check_at(ip(1), start) else {
|
||||
panic!("sixth request within the same instant must be limited");
|
||||
};
|
||||
assert_eq!(throttle.retry_after_secs, 1);
|
||||
assert_eq!(throttle.reset_after_secs, 5);
|
||||
|
||||
// One refill interval restores exactly one token.
|
||||
assert_eq!(limiter.check_at(ip(1), start + Duration::from_secs(1)), RateLimitDecision::Allowed);
|
||||
assert!(matches!(
|
||||
limiter.check_at(ip(1), start + Duration::from_secs(1)),
|
||||
RateLimitDecision::Limited(_)
|
||||
));
|
||||
|
||||
// A full window restores the full burst.
|
||||
let later = start + Duration::from_secs(61);
|
||||
for _ in 0..5 {
|
||||
assert_eq!(limiter.check_at(ip(1), later), RateLimitDecision::Allowed);
|
||||
}
|
||||
assert!(matches!(limiter.check_at(ip(1), later), RateLimitDecision::Limited(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clients_have_independent_buckets() {
|
||||
let limiter = RateLimiter::new(quota(60, 1));
|
||||
let now = Instant::now();
|
||||
assert_eq!(limiter.check_at(ip(1), now), RateLimitDecision::Allowed);
|
||||
assert!(matches!(limiter.check_at(ip(1), now), RateLimitDecision::Limited(_)));
|
||||
assert_eq!(limiter.check_at(ip(2), now), RateLimitDecision::Allowed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_clients_stay_bounded_under_distinct_ip_flood() {
|
||||
let limiter = RateLimiter::with_limits(quota(60, 1), 4, 64);
|
||||
let now = Instant::now();
|
||||
for i in 0..10_000u32 {
|
||||
limiter.check_at(IpAddr::V4(Ipv4Addr::from(i)), now);
|
||||
}
|
||||
assert!(
|
||||
limiter.tracked_clients() <= 64,
|
||||
"tracked {} clients, cap is 64",
|
||||
limiter.tracked_clients()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refilled_idle_buckets_are_swept() {
|
||||
let limiter = RateLimiter::with_limits(quota(60, 5), 1, 100);
|
||||
let start = Instant::now();
|
||||
limiter.check_at(ip(1), start);
|
||||
assert_eq!(limiter.tracked_clients(), 1);
|
||||
|
||||
// After burst/rate = 5s the bucket is full again; the next request past
|
||||
// the cleanup deadline sweeps it.
|
||||
let after_sweep = start + CLEANUP_INTERVAL + Duration::from_secs(6);
|
||||
limiter.check_at(ip(2), after_sweep);
|
||||
assert_eq!(limiter.tracked_clients(), 1, "idle refilled bucket must be dropped");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_hammering_never_exceeds_burst() {
|
||||
let limiter = Arc::new(RateLimiter::new(quota(60, 100)));
|
||||
let now = Instant::now();
|
||||
let allowed = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..8 {
|
||||
let limiter = Arc::clone(&limiter);
|
||||
let allowed = Arc::clone(&allowed);
|
||||
scope.spawn(move || {
|
||||
for _ in 0..50 {
|
||||
if limiter.check_at(ip(9), now) == RateLimitDecision::Allowed {
|
||||
allowed.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// 8 threads × 50 attempts = 400 attempts at the same instant: exactly
|
||||
// the burst may pass, regardless of interleaving.
|
||||
assert_eq!(allowed.load(std::sync::atomic::Ordering::Relaxed), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exempt_paths_cover_probes_internode_and_console_only() {
|
||||
for path in [
|
||||
"/health",
|
||||
"/health/ready",
|
||||
"/minio/health/live",
|
||||
"/minio/health/cluster/read",
|
||||
"/profile/cpu",
|
||||
"/favicon.ico",
|
||||
"/rustfs/rpc/anything",
|
||||
"/node_service.NodeService/Ping",
|
||||
"/rustfs/console/index.html",
|
||||
] {
|
||||
assert!(is_rate_limit_exempt_path(path), "{path} must be exempt");
|
||||
}
|
||||
for path in [
|
||||
"/bucket/object",
|
||||
"/",
|
||||
"/rustfs/admin/v3/info",
|
||||
"/minio/admin/v3/info",
|
||||
"/healthy-bucket/object", // prefix boundary: not /health
|
||||
"/iceberg/v1/config",
|
||||
] {
|
||||
assert!(!is_rate_limit_exempt_path(path), "{path} must be limited");
|
||||
}
|
||||
}
|
||||
|
||||
// ---- service-level tests ----
|
||||
|
||||
#[derive(Clone)]
|
||||
struct OkService;
|
||||
|
||||
impl<ReqBody> Service<Request<ReqBody>> for OkService {
|
||||
type Response = Response<BoxBody>;
|
||||
type Error = std::convert::Infallible;
|
||||
type Future = Ready<Result<Self::Response, Self::Error>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, _req: Request<ReqBody>) -> Self::Future {
|
||||
let body: BoxBody = Full::new(Bytes::from_static(b"ok"))
|
||||
.map_err(|e| -> BoxError { Box::new(e) })
|
||||
.boxed_unsync();
|
||||
ready(Ok(Response::new(body)))
|
||||
}
|
||||
}
|
||||
|
||||
fn service_with_quota(rpm: u32, burst: u32) -> RateLimitService<OkService> {
|
||||
RateLimitLayer::new(quota(rpm, burst)).layer(OkService)
|
||||
}
|
||||
|
||||
fn request_from(peer: IpAddr, path: &str) -> Request<()> {
|
||||
let mut req = Request::builder().uri(path).body(()).expect("request");
|
||||
req.extensions_mut().insert(RemoteAddr(SocketAddr::new(peer, 51000)));
|
||||
req
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn over_limit_returns_429_with_retry_after_and_ratelimit_headers() {
|
||||
let mut service = service_with_quota(60, 2);
|
||||
|
||||
for _ in 0..2 {
|
||||
let resp = service.call(request_from(ip(1), "/bucket/object")).await.expect("ok");
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
let mut req = request_from(ip(1), "/bucket/object");
|
||||
req.headers_mut().insert(X_REQUEST_ID, HeaderValue::from_static("req-123"));
|
||||
let resp = service.call(req).await.expect("ok");
|
||||
assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(resp.headers().get(http::header::RETRY_AFTER).and_then(|v| v.to_str().ok()), Some("1"));
|
||||
assert_eq!(resp.headers().get(X_RATE_LIMIT_LIMIT).and_then(|v| v.to_str().ok()), Some("60"));
|
||||
assert_eq!(resp.headers().get(X_RATE_LIMIT_REMAINING).and_then(|v| v.to_str().ok()), Some("0"));
|
||||
assert!(resp.headers().contains_key(X_RATE_LIMIT_RESET));
|
||||
assert_eq!(resp.headers().get(X_REQUEST_ID).and_then(|v| v.to_str().ok()), Some("req-123"));
|
||||
assert_eq!(
|
||||
resp.headers().get(http::header::CONTENT_TYPE).and_then(|v| v.to_str().ok()),
|
||||
Some("application/xml")
|
||||
);
|
||||
|
||||
let body = resp.into_body().collect().await.expect("body").to_bytes();
|
||||
let body = String::from_utf8_lossy(&body);
|
||||
assert!(body.contains("<Code>TooManyRequests</Code>"), "body: {body}");
|
||||
assert!(body.contains("<RequestId>req-123</RequestId>"), "body: {body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hostile_request_id_is_not_reflected_into_the_429_body() {
|
||||
let mut service = service_with_quota(60, 1);
|
||||
let _ = service.call(request_from(ip(3), "/bucket/object")).await.expect("ok");
|
||||
|
||||
let mut req = request_from(ip(3), "/bucket/object");
|
||||
req.headers_mut()
|
||||
.insert(X_REQUEST_ID, HeaderValue::from_static("<Code>evil</Code>"));
|
||||
let resp = service.call(req).await.expect("ok");
|
||||
assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
|
||||
let body = resp.into_body().collect().await.expect("body").to_bytes();
|
||||
let body = String::from_utf8_lossy(&body);
|
||||
assert!(!body.contains("evil"), "client-controlled request id must not be reflected: {body}");
|
||||
assert!(!body.contains("<RequestId>"), "malformed id must be omitted entirely: {body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exempt_paths_are_never_limited() {
|
||||
let mut service = service_with_quota(60, 1);
|
||||
for _ in 0..5 {
|
||||
let resp = service.call(request_from(ip(1), "/health/ready")).await.expect("ok");
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forwarded_for_header_cannot_choose_the_bucket() {
|
||||
let mut service = service_with_quota(60, 1);
|
||||
|
||||
// Same socket peer, rotating spoofed headers: still one bucket.
|
||||
let mut first = request_from(ip(1), "/bucket/object");
|
||||
first
|
||||
.headers_mut()
|
||||
.insert("x-forwarded-for", HeaderValue::from_static("10.0.0.1"));
|
||||
assert_eq!(service.call(first).await.expect("ok").status(), StatusCode::OK);
|
||||
|
||||
let mut second = request_from(ip(1), "/bucket/object");
|
||||
second
|
||||
.headers_mut()
|
||||
.insert("x-forwarded-for", HeaderValue::from_static("10.0.0.2"));
|
||||
second.headers_mut().insert("x-real-ip", HeaderValue::from_static("10.0.0.3"));
|
||||
assert_eq!(service.call(second).await.expect("ok").status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validated_client_info_takes_precedence_over_socket_peer() {
|
||||
let mut service = service_with_quota(60, 1);
|
||||
|
||||
// Two clients behind the same trusted proxy socket: distinct buckets.
|
||||
for client in [ip(10), ip(11)] {
|
||||
let mut req = request_from(ip(1), "/bucket/object");
|
||||
req.extensions_mut().insert(ClientInfo::direct(SocketAddr::new(client, 443)));
|
||||
assert_eq!(service.call(req).await.expect("ok").status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
// And the validated identity is throttled independently of the socket.
|
||||
let mut req = request_from(ip(2), "/bucket/object");
|
||||
req.extensions_mut().insert(ClientInfo::direct(SocketAddr::new(ip(10), 443)));
|
||||
assert_eq!(service.call(req).await.expect("ok").status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_client_identity_fails_open() {
|
||||
let mut service = service_with_quota(60, 1);
|
||||
for _ in 0..3 {
|
||||
let req = Request::builder().uri("/bucket/object").body(()).expect("request");
|
||||
assert_eq!(service.call(req).await.expect("ok").status(), StatusCode::OK);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn env_constructor_defaults_to_disabled() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, None::<&str>),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_RPM, None),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BURST, None),
|
||||
],
|
||||
|| {
|
||||
assert!(api_rate_limit_layer_from_env().is_none());
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn env_constructor_requires_enable_and_nonzero_rpm() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_RPM, None),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BURST, None),
|
||||
],
|
||||
|| {
|
||||
assert!(api_rate_limit_layer_from_env().is_none(), "enabled with rpm=0 stays inactive");
|
||||
},
|
||||
);
|
||||
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, Some("false")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_RPM, Some("600")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BURST, None),
|
||||
],
|
||||
|| {
|
||||
assert!(api_rate_limit_layer_from_env().is_none(), "rpm without enable stays inactive");
|
||||
},
|
||||
);
|
||||
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_RPM, Some("600")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BURST, Some("50")),
|
||||
],
|
||||
|| {
|
||||
let layer = api_rate_limit_layer_from_env().expect("enabled with rpm");
|
||||
assert_eq!(
|
||||
layer.quota(),
|
||||
RateLimitQuota {
|
||||
requests_per_minute: 600,
|
||||
burst: 50
|
||||
}
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,7 +88,6 @@ checked_files=(
|
||||
"crates/protocols/src/swift/container.rs"
|
||||
"crates/protocols/src/swift/object.rs"
|
||||
"crates/protocols/src/swift/formpost.rs"
|
||||
"crates/protocols/src/swift/ratelimit.rs"
|
||||
"crates/protocols/src/swift/expiration.rs"
|
||||
"crates/protocols/src/swift/cors.rs"
|
||||
"crates/protocols/src/swift/quota.rs"
|
||||
|
||||
Reference in New Issue
Block a user