mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-01 19:12:14 +00:00
houseme
8218248000
* fix(hotpath): pin mimalloc allocator backend * test(hotpath): verify mimalloc allocator backend Co-Authored-By: heihutu <heihutu@gmail.com> * chore(hotpath): document unsafe allocator tests Co-Authored-By: heihutu <heihutu@gmail.com> * feat(kms): record real cache hit, miss and eviction metrics (#5531) * feat(kms): record real cache hit, miss and eviction metrics The metadata cache reported (entry_count, 0) because moka exposes no hit or miss counts, so the miss half of every cache report was a constant. Track lookups and removals in the cache itself: hit/miss counters on the lookup path, a moka eviction listener classifying removals by cause, and an entry gauge refreshed whenever the entry set changes. The counters are exported through the metrics facade under the rustfs_kms_ prefix with static label values only, matching the operation-policy metrics, and are also returned as a KmsCacheStats snapshot in place of the old tuple. Cache semantics are unchanged: capacity, TTL and invalidation points are the same, and remove now flushes pending maintenance so the gauge and the removal notification describe the cache the caller sees. Refs rustfs/backlog#1584 * fix(kms): report real cache counters through the admin status API KmsStatusResponse.cache_stats mapped the old (entry_count, 0) tuple onto hit_count and miss_count, so operators polling KMS status read the entry count as a hit count and a miss count that was always zero. Map the fields to the counters they claim to be, and add entry_count and eviction_count as additive, defaulted fields so the entry number that hit_count used to carry is still available. Refs rustfs/backlog#1584 * fix(kms): refresh the cache entry gauge on lookup misses The entry gauge was published only from the write paths, so an entry dropped by TTL expiry left `rustfs_kms_metadata_cache_entries` reporting a population that no longer existed until the next put, remove or clear. A cache that goes quiet — entries ageing out with no further writes — kept over-reporting indefinitely. Republish the gauge from the lookup path when the lookup misses. A miss is where expiry surfaces, and moka reaps expired entries in the maintenance it runs during that same lookup, so the count read afterwards reflects the reaping. Hits stay free of the extra work. * docs(kms): correct the entry gauge convergence claim on the miss path The comment on the miss-path gauge refresh said moka reaps expired entries in the maintenance it runs on that same lookup. It does not: `should_apply_reads` is gated on a full read log or an elapsed housekeeping interval, so the removal that decrements `entry_count` and reaches the eviction listener may land on a later lookup. The behaviour and the test are unchanged — the gauge still converges, and the test drives `run_pending_tasks` explicitly rather than riding on that interval. Only the stated guarantee was wrong, so say interval instead of same-lookup and record why forcing maintenance on the read path was not the trade taken. * chore(deps): refresh cargo dependencies Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
382 lines
15 KiB
TOML
382 lines
15 KiB
TOML
# Copyright 2024 RustFS Team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
[workspace]
|
|
members = [
|
|
"rustfs", # Core file system implementation
|
|
"crates/audit", # Audit target management system with multi-target fan-out
|
|
"crates/checksums", # client checksums
|
|
"crates/common", # Shared utilities and data structures
|
|
"crates/data-usage", # Shared data usage models and algorithms
|
|
"crates/config", # Configuration management
|
|
"crates/credentials", # Credential management system
|
|
"crates/crypto", # Cryptography and security features
|
|
"crates/ecstore", # Erasure coding storage implementation
|
|
"crates/e2e_test", # End-to-end test suite
|
|
"crates/filemeta", # File metadata management
|
|
"crates/heal", # Erasure set and object healing
|
|
"crates/iam", # Identity and Access Management
|
|
"crates/keystone", # OpenStack Keystone integration
|
|
"crates/lifecycle", # Lifecycle rule evaluation contracts
|
|
"crates/kms", # Key Management Service
|
|
"crates/lock", # Distributed locking implementation
|
|
"crates/log-analyzer", # Offline log fault-analysis core (rustfs diagnose)
|
|
"crates/madmin", # Management dashboard and admin API interface
|
|
"crates/notify", # Notification system for events
|
|
"crates/obs", # Observability utilities
|
|
"crates/object-capacity", # Capacity scan and refresh core
|
|
"crates/object-data-cache", # Long-term object body cache engine
|
|
"crates/policy", # Policy management
|
|
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
|
|
"crates/protos", # Protocol buffer definitions
|
|
"crates/rio", # Rust I/O utilities and abstractions
|
|
"crates/rio-v2", # Next-generation Rust I/O compatibility layer
|
|
"crates/replication", # Replication contracts and wire formats
|
|
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
|
"crates/s3-types", # S3 event type definitions
|
|
"crates/s3-ops", # S3 operation definitions and mapping
|
|
"crates/s3select-api", # S3 Select API interface
|
|
"crates/s3select-query", # S3 Select query engine
|
|
"crates/scanner", # Scanner for data integrity checks and health monitoring
|
|
"crates/security-governance", # Security governance contracts
|
|
"crates/extension-schema", # Extension schema contracts
|
|
"crates/signer", # client signer
|
|
"crates/storage-api", # Storage API contracts
|
|
"crates/test-utils", # Shared test bootstrap helpers (dev-dependency only)
|
|
"crates/targets", # Target-specific configurations and utilities
|
|
"crates/trusted-proxies", # Trusted proxies management
|
|
"crates/tls-runtime", # Project-wide TLS runtime foundation
|
|
"crates/utils", # Utility functions and helpers
|
|
"crates/io-metrics", # I/O metrics collection for performance analysis
|
|
"crates/io-core", # Buffered Bytes, mmap-then-copy, and aligned pread I/O helpers
|
|
"crates/zip", # ZIP file handling and compression
|
|
]
|
|
resolver = "3"
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
license = "Apache-2.0"
|
|
repository = "https://github.com/rustfs/rustfs"
|
|
rust-version = "1.97.1"
|
|
version = "1.0.0-beta.12"
|
|
homepage = "https://rustfs.com"
|
|
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
|
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
|
categories = ["web-programming", "development-tools", "filesystem", "network-programming"]
|
|
authors = ["RustFS Team"]
|
|
|
|
[workspace.lints.rust]
|
|
unsafe_code = "deny"
|
|
|
|
[workspace.lints.clippy]
|
|
all = "warn"
|
|
needless_collect = "warn"
|
|
redundant_clone = "warn"
|
|
|
|
[workspace.dependencies]
|
|
# RustFS Internal Crates
|
|
rustfs = { path = "./rustfs", version = "1.0.0-beta.12" }
|
|
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.12" }
|
|
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.12" }
|
|
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.12" }
|
|
rustfs-common = { path = "crates/common", version = "1.0.0-beta.12" }
|
|
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.12" }
|
|
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.12" }
|
|
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.12" }
|
|
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.12" }
|
|
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.12" }
|
|
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.12" }
|
|
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.12" }
|
|
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.12" }
|
|
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.12" }
|
|
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.12" }
|
|
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.12" }
|
|
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.12" }
|
|
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.12" }
|
|
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.12" }
|
|
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.12" }
|
|
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.12" }
|
|
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.12" }
|
|
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.12" }
|
|
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.12" }
|
|
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.12" }
|
|
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.12" }
|
|
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.12" }
|
|
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.12" }
|
|
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.12" }
|
|
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.12" }
|
|
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.12" }
|
|
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.12" }
|
|
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.12" }
|
|
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.12" }
|
|
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.12" }
|
|
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.12" }
|
|
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.12" }
|
|
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.12" }
|
|
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.12" }
|
|
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.12" }
|
|
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.12" }
|
|
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.12" }
|
|
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.12" }
|
|
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.12" }
|
|
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.12" }
|
|
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.12" }
|
|
|
|
# Async Runtime and Networking
|
|
async-channel = "2.5.0"
|
|
async_zip = { default-features = false, version = "0.0.18" }
|
|
mysql_async = { default-features = false, version = "0.37" }
|
|
async-compression = { version = "0.4.43" }
|
|
async-recursion = "1.1.1"
|
|
async-trait = "0.1.91"
|
|
async-nats = { version = "0.50.0", default-features = false }
|
|
axum = "0.8.9"
|
|
futures = "0.3.33"
|
|
futures-core = "0.3.33"
|
|
futures-lite = "2.6.1"
|
|
futures-util = "0.3.33"
|
|
pollster = "1.0.1"
|
|
pulsar = { default-features = false, version = "6.8.0" }
|
|
lapin = { default-features = false, version = "4.10.0" }
|
|
hyper = { version = "1.11.0" }
|
|
hyper-rustls = { default-features = false, version = "0.27.9" }
|
|
hyper-util = { version = "0.1.20" }
|
|
http = "1.5.0"
|
|
http-body = "1.1.0"
|
|
http-body-util = "0.1.4"
|
|
minlz = "1.2.3"
|
|
reqwest = "0.13.4"
|
|
rustfs-kafka-async = { version = "1.2.0" }
|
|
socket2 = { version = "0.6.5" }
|
|
tokio = { version = "1.53.1" }
|
|
tokio-rustls = { default-features = false, version = "0.26.4" }
|
|
tokio-stream = { version = "0.1.19" }
|
|
tokio-test = "0.4.5"
|
|
tokio-util = { version = "0.7.19" }
|
|
tonic = { version = "0.14.6" }
|
|
tonic-prost = { version = "0.14.6" }
|
|
tonic-prost-build = { version = "0.14.6" }
|
|
tower = { version = "0.5.3" }
|
|
tower-http = { version = "0.7.0" }
|
|
|
|
# Serialization and Data Formats
|
|
apache-avro = "0.21.0"
|
|
bytes = { version = "1.12.1" }
|
|
bytesize = "2.6.0"
|
|
byteorder = "1.5.0"
|
|
flatbuffers = "25.12.19"
|
|
form_urlencoded = "1.2.2"
|
|
prost = "0.14.4"
|
|
quick-xml = "0.41.0"
|
|
rmp = { version = "0.8.15" }
|
|
rmp-serde = { version = "1.3.1" }
|
|
serde = { version = "1.0.229" }
|
|
serde_json = { version = "1.0.151" }
|
|
serde_urlencoded = "0.7.1"
|
|
|
|
# Cryptography and Security
|
|
# NOTE: These direct cryptography dependencies use RC versions because the
|
|
# matching stable releases are not available yet, while previous stable lines
|
|
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
|
|
# releases.
|
|
aes-gcm = { version = "=0.11.0" }
|
|
argon2 = { version = "=0.6.0-rc.8" }
|
|
blake2 = "=0.11.0-rc.6"
|
|
chacha20poly1305 = { version = "=0.11.0" }
|
|
crc-fast = "1.10.0"
|
|
hmac = { version = "0.13.0" }
|
|
jsonwebtoken = { version = "11.0.0" }
|
|
openidconnect = { default-features = false, version = "4.0" }
|
|
pbkdf2 = "0.13.0"
|
|
rsa = { version = "=0.10.0-rc.18" }
|
|
rustls = { default-features = false, version = "0.23.43" }
|
|
rustls-native-certs = "0.8"
|
|
rustls-pki-types = "1.15.1"
|
|
sha1 = "0.11.0"
|
|
sha2 = "0.11.0"
|
|
subtle = "2.6"
|
|
zeroize = { version = "1.9.0" }
|
|
|
|
# Time and Date
|
|
chrono = { version = "0.4.45" }
|
|
humantime = "2.4.0"
|
|
jiff = { version = "0.2.35" }
|
|
time = { version = "0.3.54" }
|
|
|
|
# Database
|
|
deadpool-postgres = { version = "0.14" }
|
|
tokio-postgres = { default-features = false, version = "0.7.18" }
|
|
tokio-postgres-rustls = "0.14.0"
|
|
|
|
# Utilities and Tools
|
|
anyhow = "1.0.104"
|
|
arc-swap = "1.9.2"
|
|
astral-tokio-tar = "0.6.4"
|
|
atoi = "3.1.0"
|
|
atomic_enum = "0.3.0"
|
|
aws-config = { version = "1.10.1" }
|
|
aws-credential-types = { version = "1.3.0" }
|
|
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
|
aws-sdk-s3 = { default-features = false, version = "1.140.0" }
|
|
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
|
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
|
aws-smithy-runtime-api = { version = "1.14.0" }
|
|
aws-smithy-types = { version = "1.6.1" }
|
|
base64 = "0.23.0"
|
|
base64-simd = "0.8.0"
|
|
brotli = "8.0.4"
|
|
clap = { version = "4.6.5" }
|
|
const-str = { version = "1.1.0" }
|
|
convert_case = "0.11.0"
|
|
criterion = { version = "0.8" }
|
|
crossbeam-queue = "0.3.13"
|
|
crossbeam-channel = "0.5.16"
|
|
crossbeam-deque = "0.8.7"
|
|
crossbeam-utils = "0.8.22"
|
|
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
|
|
#datafusion = { default-features = false, version = "54.1.0" }
|
|
derive_builder = "0.20.2"
|
|
enumset = "1.1.14"
|
|
faster-hex = "0.10.0"
|
|
flate2 = "1.1.9"
|
|
glob = "0.3.4"
|
|
google-cloud-storage = "1.17.0"
|
|
google-cloud-auth = "1.15.0"
|
|
hashbrown = { version = "0.17.1" }
|
|
hex = "0.4.3"
|
|
hex-simd = "0.8.0"
|
|
highway = { version = "1.3.0" }
|
|
hostname = "0.4.2"
|
|
ipnetwork = { version = "0.21.1" }
|
|
lazy_static = "1.5.0"
|
|
libc = "0.2.189"
|
|
libsystemd = "0.7.2"
|
|
local-ip-address = "0.6.13"
|
|
memmap2 = "0.9.11"
|
|
lz4 = "1.28.1"
|
|
matchit = "0.9.2"
|
|
md-5 = "0.11.0"
|
|
mime_guess = "2.0.5"
|
|
moka = { version = "0.12.15" }
|
|
netif = "0.1.6"
|
|
num_cpus = { version = "1.17.0" }
|
|
nvml-wrapper = "0.12.1"
|
|
parking_lot = "0.12.5"
|
|
path-absolutize = "4.0.1"
|
|
path-clean = "1.0.1"
|
|
percent-encoding = "2.3.2"
|
|
pin-project-lite = "0.2.17"
|
|
pretty_assertions = "1.4.1"
|
|
rand = { version = "0.10.2" }
|
|
ratelimit = "0.10.1"
|
|
rayon = "1.12.0"
|
|
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
|
reed-solomon-simd = "3.1.0"
|
|
regex = { version = "1.13.1" }
|
|
rumqttc = { package = "rumqttc-next", version = "0.33.3" }
|
|
redis = { version = "1.5.0" }
|
|
rustify = { version = "0.7", default-features = false }
|
|
rustix = { version = "1.1.4" }
|
|
rust-embed = { version = "8.12.0" }
|
|
rustc-hash = { version = "2.1.3" }
|
|
s3s = { git = "https://github.com/cxymds/s3s.git", rev = "fe3941d91fa1c69956f209a9145995c9f0235bff" }
|
|
serial_test = "4.0.1"
|
|
shadow-rs = { default-features = false, version = "2.0.0" }
|
|
siphasher = "1.0.3"
|
|
smallvec = { version = "1.15.2" }
|
|
smartstring = "1.0.1"
|
|
snap = "1.1.2"
|
|
starshard = { version = "2.2.2" }
|
|
strum = { version = "0.28.0" }
|
|
sysinfo = "0.39.6"
|
|
temp-env = "0.3.6"
|
|
tempfile = "3.27.0"
|
|
test-case = "3.3.1"
|
|
thiserror = "2.0.19"
|
|
tracing = { version = "0.1.44" }
|
|
tracing-appender = "0.2.5"
|
|
tracing-core = "0.1.36"
|
|
tracing-error = "0.2.1"
|
|
tracing-opentelemetry = { version = "0.33" }
|
|
tracing-subscriber = { version = "0.3.23" }
|
|
transform-stream = "0.3.1"
|
|
url = "2.5.8"
|
|
urlencoding = "2.1.3"
|
|
uuid = { version = "1.24.0" }
|
|
vaultrs = { version = "0.8.0" }
|
|
tar = "0.4.46"
|
|
walkdir = "2.5.0"
|
|
winapi-util = "0.1.11"
|
|
windows = { version = "0.62.2" }
|
|
windows-sys = "0.61.2"
|
|
xxhash-rust = { version = "0.8.18" }
|
|
zip = "8.6.0"
|
|
zstd = "0.13.3"
|
|
|
|
# Observability and Metrics
|
|
metrics = "0.24.6"
|
|
dial9-tokio-telemetry = "0.3"
|
|
opentelemetry = { version = "0.32.0" }
|
|
opentelemetry-appender-tracing = { version = "0.32.0" }
|
|
opentelemetry-otlp = { version = "0.32.0" }
|
|
opentelemetry-proto = { version = "0.32.0", default-features = false, features = ["metrics", "gen-tonic-messages"] }
|
|
opentelemetry_sdk = { version = "0.32.1" }
|
|
opentelemetry-semantic-conventions = { version = "0.32.1" }
|
|
opentelemetry-stdout = { version = "0.32.0" }
|
|
pyroscope = { version = "2.1.1" }
|
|
|
|
# FTP and SFTP
|
|
libunftp = { version = "0.23.0" }
|
|
unftp-core = "0.1.0"
|
|
suppaftp = { version = "10.0.1" }
|
|
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
|
russh = { version = "0.62.5" }
|
|
russh-sftp = "2.3.0"
|
|
|
|
# WebDAV
|
|
dav-server = "0.11.0"
|
|
|
|
# Performance Analysis and Memory Profiling
|
|
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13" }
|
|
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13", features = ["extended"] }
|
|
hotpath = { version = "0.22.0", default-features = false }
|
|
# Snapshot testing for output format regression detection
|
|
insta = { version = "1.48" }
|
|
|
|
[workspace.metadata.cargo-shear]
|
|
ignored = ["rustfs"]
|
|
|
|
[profile.dev]
|
|
# Full debuginfo roughly doubles compile+link time and produces multi-GB
|
|
# target dirs (slow CI cache save/restore, slow artifact upload). Line tables
|
|
# keep meaningful backtraces. For local debugging with full debuginfo, run:
|
|
# CARGO_PROFILE_DEV_DEBUG=full cargo build
|
|
debug = "line-tables-only"
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
lto = "thin"
|
|
codegen-units = 1
|
|
debug = 0
|
|
split-debuginfo = "off"
|
|
strip = "symbols"
|
|
|
|
[profile.production]
|
|
inherits = "release"
|
|
|
|
[profile.profiling]
|
|
inherits = "release"
|
|
debug = true
|
|
strip = "none"
|