gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-07 05:43:14 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
4542d4060fe9cfcb61046c1827de967f2a1af990
rustfs/rustfs/Cargo.toml
T
houseme eebd16d8a4 feat(cache): add object data cache engine and app flow (#4187) 
* feat(cache): add object data cache engine

* feat(cache): wire app-layer object cache flow

* refactor(cache): streamline app-layer cache flow

* refactor(cache): tighten cache flow internals

* refactor: address final clippy cleanup

* chore(deps): update quick-xml to 0.41.0

* feat(cache): wire object data cache env config

* fix(cache): gate materialize fill by cache plan

* chore(cache): add object data cache benchmark gate

* fix(cache): guard object cache fill size mismatches

* refactor(cache): streamline object cache body planning

* fix(cache): align object cache rollout config

* test(cache): cover buffered object cache benchmark

* test(cache): isolate object cache benchmark metrics

* test(cache): mark materialize rollout experimental

* test(cache): tighten object cache benchmark gate

* fix(cache): address review findings for object data cache

- singleflight: clean up leader entry on cancellation (Drop impl) so a dropped GET future can no longer wedge all subsequent fills for the same key; switch the fill map to a std Mutex and add a regression test

- adapter: honor RUSTFS_OBJECT_DATA_CACHE_ENABLE=true by defaulting to hit_only when no explicit mode is set (explicit mode still wins)

- planner: treat nil version UUIDs as "no value" per repo convention so unversioned objects key under the canonical "null" instead of fragmenting the key space

- multipart: invalidate the object cache on the quota-exceeded rollback delete after complete-multipart, closing a stale-cache window

- layering: move the disabled-cache fallback into app::context and drop the new infra->app layer-dependency baseline entry

* fix(cache): close invalidation races and drop full-cache scan on writes

- index: make identity-index insert/remove/prune atomic via starshard compute_if_present/compute_if_absent so concurrent fills can no longer drop each other's keys (lost keys made entries unreachable to invalidation until TTL); add a concurrency regression test

- fill: register the key in the identity index before the entry becomes visible in the cache and re-check the index afterwards, undoing the fill when an invalidation raced in between (new skipped_invalidation_race fill result)

- invalidate: with the index now authoritative, remove the full-cache iter() fallback that made every PUT/DELETE of a never-cached object O(total cache entries) (two scans per PUT, 2N per batch delete)

- materialize-fill: fail the GET instead of falling back to the partially consumed stream after a mid-read error (the fallback would send a body missing its prefix under a full-length Content-Length), and log the same size-mismatch warning as the sibling buffering paths

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(storage): fix media-dependent buffer clamp expectation

test_concurrency_manager_multi_factor_strategy_buffer_clamp asserted media_cap.min(MI_B), but the implementation's final safety clamp is [32KiB, media_cap.max(MI_B)] — deliberately so a media cap above 1MiB (NVMe's 2MiB default) stays effective. The test only passed on machines detected as SSD/Unknown (cap == 1MiB) and failed on NVMe-backed CI runners with 2MiB != 1MiB. Assert the media cap itself, which is what the strategy actually guarantees on every environment.

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(storage): format buffer clamp assertion

* chore(logging): update tier guardrail path

---------

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <cxymds@gmail.com>
Co-authored-by: overtrue <anzhengchao@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
2026-07-03 18:11:14 +08:00

227 lines
7.4 KiB
TOML
Raw Blame History

# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
homepage.workspace = true
default-run = "rustfs"
description = "RustFS is a high-performance, distributed file system designed for modern cloud-native applications, providing efficient data storage and retrieval with advanced features like S3 Select, IAM, and policy management."
keywords.workspace = true
categories.workspace = true
documentation = "https://docs.rustfs.com/"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
name = "rustfs"
path = "src/lib.rs"
[[bin]]
name = "rustfs"
path = "src/main.rs"
[[bin]]
name = "manual-test-dial9"
path = "tests/manual/test_dial9.rs"
test = false
bench = false
[[bench]]
name = "s3_operations"
harness = false
required-features = ["manual-test-runners"]
[features]
default = ["ftps", "webdav"]
metrics-gpu = ["rustfs-obs/gpu"]
ftps = ["rustfs-protocols/ftps"]
swift = ["rustfs-protocols/swift"]
webdav = ["rustfs-protocols/webdav"]
sftp = ["rustfs-protocols/sftp"]
license = []
io-scheduler-debug = [] # Enable debug information in I/O scheduler
tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only)
full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp"]
manual-test-runners = []
rio-v2 = ["rustfs-ecstore/rio-v2"]
pyroscope = ["rustfs-obs/pyroscope"]
[lints]
workspace = true
[dependencies]
# RustFS Internal Crates
rustfs-heal = { workspace = true }
rustfs-audit = { workspace = true }
rustfs-common = { workspace = true }
rustfs-config = { workspace = true, features = ["constants", "notify", "server-config-model"] }
rustfs-crypto = { workspace = true }
rustfs-credentials = { workspace = true }
rustfs-ecstore = { workspace = true }
rustfs-filemeta.workspace = true
rustfs-iam = { workspace = true }
rustfs-keystone = { workspace = true }
rustfs-kms = { workspace = true }
rustfs-lock.workspace = true
rustfs-madmin = { workspace = true }
rustfs-notify = { workspace = true }
rustfs-obs = { workspace = true }
rustfs-policy = { workspace = true }
rustfs-protocols = { workspace = true }
rustfs-protos = { workspace = true }
rustfs-rio = { workspace = true }
rustfs-replication = { workspace = true }
rustfs-s3-types = { workspace = true }
rustfs-s3-ops = { workspace = true }
rustfs-security-governance = { workspace = true }
rustfs-data-usage = { workspace = true }
rustfs-s3select-api = { workspace = true }
rustfs-s3select-query = { workspace = true }
rustfs-extension-schema = { workspace = true }
rustfs-targets = { workspace = true }
rustfs-storage-api = { workspace = true }
rustfs-tls-runtime = { workspace = true }
rustfs-trusted-proxies = { workspace = true }
rustfs-utils = { workspace = true, features = ["full"] }
rustfs-zip = { workspace = true }
rustfs-io-core = { workspace = true }
rustfs-io-metrics = { workspace = true }
rustfs-object-capacity = { workspace = true }
rustfs-object-data-cache = { workspace = true }
rustfs-concurrency = { workspace = true }
rustfs-scanner = { workspace = true }
tempfile = { workspace = true }
starshard = { workspace = true }
# Async Runtime and Networking
async-trait = { workspace = true }
axum.workspace = true
futures.workspace = true
futures-lite.workspace = true
futures-util.workspace = true
hyper.workspace = true
hyper-util.workspace = true
http.workspace = true
http-body.workspace = true
http-body-util.workspace = true
reqwest = { workspace = true }
socket2 = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util"] }
tokio-rustls = { workspace = true }
aws-sdk-s3 = { workspace = true }
tokio-stream.workspace = true
tokio-util.workspace = true
tonic = { workspace = true }
tower.workspace = true
tower-http = { workspace = true, features = ["trace", "compression-full", "cors", "catch-panic", "timeout", "limit", "request-id", "add-extension"] }
# Serialization and Data Formats
apache-avro = { workspace = true }
bytes = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
flatbuffers.workspace = true
rmp-serde.workspace = true
rustfs-signer.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_urlencoded = { workspace = true }
# Cryptography and Security
rustls = { workspace = true }
rustls-pki-types = { workspace = true }
subtle = { workspace = true }
jiff = { workspace = true }
time = { workspace = true, features = ["parsing", "formatting", "serde"] }
# Utilities and Tools
astral-tokio-tar = { workspace = true }
atoi = { workspace = true }
atomic_enum = { workspace = true }
async_zip = { workspace = true }
base64 = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
base64-simd.workspace = true
clap = { workspace = true }
const-str = { workspace = true }
datafusion = { workspace = true }
hex-simd.workspace = true
matchit = { workspace = true }
md5.workspace = true
mime_guess = { workspace = true }
percent-encoding = { workspace = true }
pin-project-lite.workspace = true
rust-embed = { workspace = true, features = ["interpolate-folder-path"] }
s3s.workspace = true
shadow-rs = { workspace = true, features = ["build", "metadata"] }
sysinfo = { workspace = true, features = ["multithread"] }
thiserror = { workspace = true }
tracing.workspace = true
url = { workspace = true }
urlencoding = { workspace = true }
uuid = { workspace = true }
zip = { workspace = true }
libc = { workspace = true }
rand = { workspace = true }
aes-gcm = { workspace = true }
chacha20poly1305 = { workspace = true }
# Observability and Metrics
metrics = { workspace = true }
opentelemetry = { workspace = true }
tracing-opentelemetry = { workspace = true }
# Data structures
hashbrown = { workspace = true }
mimalloc = { workspace = true }
[target.'cfg(target_os = "linux")'.dependencies]
libsystemd.workspace = true
# io-uring is Linux-only. Scope the feature to Linux targets.
tokio = { workspace = true, features = ["io-uring"] }
[target.'cfg(not(target_os = "windows"))'.dependencies]
libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
[dev-dependencies]
uuid = { workspace = true, features = ["v4"] }
serial_test = { workspace = true }
tempfile = { workspace = true }
aws-config = { workspace = true }
anyhow = { workspace = true }
insta = { workspace = true }
proptest = "1"
tokio = { workspace = true, features = ["test-util"] }
temp-env = { workspace = true, features = ["async_closure"] }
tracing-subscriber = { workspace = true }
opentelemetry_sdk = { workspace = true }
rsa = { workspace = true }
rcgen = { workspace = true }
criterion = { workspace = true, features = ["html_reports"] }
[build-dependencies]
http.workspace = true
futures.workspace = true
s3s = { workspace = true }
clap = { workspace = true }
hyper-util = { workspace = true, features = [
"tokio",
"server-auto",
"server-graceful",
] }
shadow-rs = { workspace = true, features = ["build"] }
Reference in New Issue View Git Blame Copy Permalink