gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 14:23:13 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
f13e98eb817e24b86c8a3d82e0afd6201dcf942e
rustfs/crates/ecstore/Cargo.toml
T
houseme fe682e16e9 feat(ecstore): run one io_uring ring per shard on each disk (backlog#1145) (#4653) 
A buffered read that hits the page cache completes inline inside
`io_uring_enter`, so the thread driving a ring performs that read's
memcpy. One ring per disk therefore capped cache-hit reads at a single
core's memory bandwidth: measured on a 16-core host, one driver thread sat
pinned at 100% CPU while throughput stayed flat at ~5 GB/s regardless of
read size, against 50 GB/s for the blocking-pool baseline.

rustfs/uring#6 taught the driver to hold N independent rings, each with its
own thread, pending table, backpressure semaphore, and eventfd. Wire it up:
`UringBackend::try_new` now calls `probe_and_start_sharded`, and
`RUSTFS_IO_URING_SHARDS` selects the count per disk.

The default is a quarter of the available parallelism clamped to `1..=4`,
because the cost is `disks × shards` driver threads (each normally blocked
in `poll(2)`). Any override is clamped to `1..=16`, so a mistyped value can
neither disable the driver (0) nor spawn threads without bound; an
unparseable value falls back to the default.

Effect (warm page cache, 16-core, rustfs/uring's concurrent_pread_bench):

  1 MiB, conc 8:    1 shard  4911 MB/s -> 8 shards 47361 MB/s (9.6x);
                    the blocking-pool baseline is 50662 MB/s
  64 KiB, conc 32:  StdBackend 153678 IOPS, p999 3030 us
                    8 shards   345402 IOPS, p999  897 us
  64 KiB, conc 128: StdBackend 135155 IOPS, p999 10716 us
                    8 shards    389047 IOPS, p999  4092 us

Sharding removes the throughput deficit *and* keeps io_uring's tail-latency
advantage, rather than trading one for the other.

Unchanged: io_uring read stays gray-off by default
(`RUSTFS_IO_URING_READ_ENABLE`), reads are byte-for-byte identical to
StdBackend, the per-disk degradation latches and probe cache (backlog#1101)
and the O_DIRECT tiered fallback (backlog#1102) all still apply. Rings stay
per-disk, so a stalled disk cannot starve another disk's rings
(backlog#1055). Bumps the rustfs-uring pin to the merged #6 commit.

Verified on a real Linux host (16-core, real io_uring): cargo clippy
--tests -D warnings clean; disk::local tests 132 passed, 0 failed —
including the existing io_uring and O_DIRECT cases now running on the
sharded driver, plus a new test covering the shard-count default, override,
and clamping.

Co-authored-by: heihutu <heihutu@gmail.com>
2026-07-10 07:41:11 +00:00

183 lines
6.0 KiB
TOML
Raw Blame History

# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-ecstore"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
homepage.workspace = true
description = "Erasure coding storage backend for RustFS, providing efficient data storage and retrieval with redundancy."
keywords = ["erasure-coding", "storage", "rustfs", "Minio", "solomon"]
categories = ["web-programming", "development-tools", "filesystem"]
documentation = "https://docs.rs/rustfs-ecstore/latest/rustfs_ecstore/"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lints]
workspace = true
[features]
default = []
rio-v2 = ["dep:rustfs-rio-v2"]
hotpath = ["dep:hotpath", "hotpath/hotpath", "rustfs-filemeta/hotpath", "rustfs-rio/hotpath"]
[dependencies]
hotpath = { workspace = true, optional = true }
rustfs-filemeta.workspace = true
rustfs-utils = { workspace = true, features = ["full"] }
rustfs-rio.workspace = true
rustfs-rio-v2 = { workspace = true, optional = true }
rustfs-signer.workspace = true
rustfs-storage-api.workspace = true
rustfs-tls-runtime.workspace = true
rustfs-checksums.workspace = true
rustfs-config = { workspace = true, features = ["constants", "notify", "audit", "server-config-model"] }
rustfs-concurrency.workspace = true
rustfs-credentials = { workspace = true }
rustfs-common.workspace = true
rustfs-policy.workspace = true
rustfs-protos.workspace = true
rustfs-replication.workspace = true
rustfs-lifecycle.workspace = true
rustfs-kms.workspace = true
rustfs-s3-types = { workspace = true }
rustfs-data-usage.workspace = true
rustfs-object-capacity.workspace = true
async-trait.workspace = true
bytes.workspace = true
byteorder = { workspace = true }
chrono.workspace = true
glob = { workspace = true }
thiserror.workspace = true
flatbuffers.workspace = true
futures.workspace = true
futures-util.workspace = true
tracing.workspace = true
tracing-opentelemetry.workspace = true
serde.workspace = true
time.workspace = true
bytesize.workspace = true
serde_json.workspace = true
quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
s3s.workspace = true
http.workspace = true
opentelemetry.workspace = true
http-body = { workspace = true }
http-body-util.workspace = true
url.workspace = true
uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
reed-solomon-erasure = { workspace = true }
reed-solomon-simd = { workspace = true }
lazy_static.workspace = true
moka = { workspace = true }
rustfs-lock.workspace = true
rustfs-io-metrics.workspace = true
regex = { workspace = true }
path-absolutize = { workspace = true }
rmp.workspace = true
rmp-serde.workspace = true
tokio-util = { workspace = true, features = ["io", "compat"] }
base64 = { workspace = true }
hmac = { workspace = true }
sha1 = { workspace = true }
sha2 = { workspace = true }
hex-simd = { workspace = true }
tempfile.workspace = true
hyper.workspace = true
hyper-util.workspace = true
hyper-rustls.workspace = true
rustls.workspace = true
rustls-pki-types.workspace = true
tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
tonic.workspace = true
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
tower.workspace = true
async-channel.workspace = true
enumset = { workspace = true }
num_cpus = { workspace = true }
rand.workspace = true
pin-project-lite.workspace = true
md-5.workspace = true
memmap2 = { workspace = true }
libc.workspace = true
rustix = { workspace = true }
rustfs-madmin.workspace = true
reqwest = { workspace = true }
aes-gcm.workspace = true
chacha20poly1305.workspace = true
aws-sdk-s3 = { workspace = true }
urlencoding = { workspace = true }
smallvec = { workspace = true }
shadow-rs.workspace = true
async-recursion.workspace = true
aws-credential-types = { workspace = true }
aws-smithy-types = { workspace = true }
aws-smithy-runtime-api = { workspace = true }
parking_lot = { workspace = true }
base64-simd.workspace = true
serde_urlencoded.workspace = true
google-cloud-storage = { workspace = true }
google-cloud-auth = { workspace = true }
aws-config = { workspace = true }
faster-hex = { workspace = true }
ratelimit = { workspace = true }
aws-smithy-http-client.workspace = true
# Observability and Metrics
metrics = { workspace = true }
# Runtime-probed io_uring read backend (backlog#1104). Linux-only, git
# dependency until rustfs-uring is production-ready and published. The guard
# scripts/check_no_tokio_io_uring.sh allows an explicit io-uring integration;
# only the tokio "io-uring" runtime feature is banned.
[target.'cfg(target_os = "linux")'.dependencies]
rustfs-uring = { git = "https://github.com/rustfs/uring", rev = "719b245f9bb24e838de8639c19858df2690f3ae1" }
[dev-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] }
criterion = { workspace = true, features = ["html_reports"] }
temp-env = { workspace = true, features = ["async_closure"] }
tracing-subscriber = { workspace = true, features = ["json"] }
serial_test = { workspace = true }
opentelemetry_sdk = { workspace = true }
proptest = "1"
rcgen.workspace = true
insta = { workspace = true }
rustfs-crypto = { workspace = true }
[build-dependencies]
shadow-rs = { workspace = true, features = ["build", "metadata"] }
[[bench]]
name = "erasure_benchmark"
harness = false
[[bench]]
name = "comparison_benchmark"
harness = false
[[bench]]
name = "rename_data_meta_benchmark"
harness = false
[[bench]]
name = "single_block_non_inline_benchmark"
harness = false
[lib]
doctest = false
Reference in New Issue View Git Blame Copy Permalink