diff --git a/.docker/compose/docker-compose.cluster.perf-round1-linux-32c64g-latency-stable.override.yml b/.docker/compose/docker-compose.cluster.perf-round1-linux-32c64g-latency-stable.override.yml new file mode 100644 index 000000000..59961fc19 --- /dev/null +++ b/.docker/compose/docker-compose.cluster.perf-round1-linux-32c64g-latency-stable.override.yml @@ -0,0 +1,56 @@ +services: + node1: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=56 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=131072 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=10 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=20 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=10 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824 + - RUSTFS_RUNTIME_WORKER_THREADS=16 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=768 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=420 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=50 + + node2: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=56 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=131072 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=10 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=20 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=10 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824 + - RUSTFS_RUNTIME_WORKER_THREADS=16 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=768 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=420 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=50 + + node3: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=56 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=131072 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=10 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=20 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=10 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824 + - RUSTFS_RUNTIME_WORKER_THREADS=16 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=768 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=420 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=50 + + node4: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=56 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=131072 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=10 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=20 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=10 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824 + - RUSTFS_RUNTIME_WORKER_THREADS=16 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=768 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=420 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=50 diff --git a/.docker/compose/docker-compose.cluster.perf-round1-linux-32c64g-throughput-max.override.yml b/.docker/compose/docker-compose.cluster.perf-round1-linux-32c64g-throughput-max.override.yml new file mode 100644 index 000000000..8a40f4ccb --- /dev/null +++ b/.docker/compose/docker-compose.cluster.perf-round1-linux-32c64g-throughput-max.override.yml @@ -0,0 +1,56 @@ +services: + node1: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=96 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=524288 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=16 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=32 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=24 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=33554432 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=50331648 + - RUSTFS_RUNTIME_WORKER_THREADS=24 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1536 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=600 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=80 + + node2: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=96 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=524288 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=16 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=32 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=24 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=33554432 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=50331648 + - RUSTFS_RUNTIME_WORKER_THREADS=24 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1536 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=600 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=80 + + node3: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=96 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=524288 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=16 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=32 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=24 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=33554432 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=50331648 + - RUSTFS_RUNTIME_WORKER_THREADS=24 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1536 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=600 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=80 + + node4: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=96 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=524288 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=16 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=32 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=24 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=33554432 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=50331648 + - RUSTFS_RUNTIME_WORKER_THREADS=24 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1536 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=600 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=80 diff --git a/.docker/compose/docker-compose.cluster.perf-round1-linux-32c64g.override.yml b/.docker/compose/docker-compose.cluster.perf-round1-linux-32c64g.override.yml new file mode 100644 index 000000000..ac973ba31 --- /dev/null +++ b/.docker/compose/docker-compose.cluster.perf-round1-linux-32c64g.override.yml @@ -0,0 +1,56 @@ +services: + node1: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=12 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=24 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=16 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=16777216 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=33554432 + - RUSTFS_RUNTIME_WORKER_THREADS=20 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=60 + + node2: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=12 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=24 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=16 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=16777216 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=33554432 + - RUSTFS_RUNTIME_WORKER_THREADS=20 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=60 + + node3: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=12 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=24 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=16 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=16777216 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=33554432 + - RUSTFS_RUNTIME_WORKER_THREADS=20 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=60 + + node4: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=12 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=24 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=16 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=16777216 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=33554432 + - RUSTFS_RUNTIME_WORKER_THREADS=20 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=60 diff --git a/.docker/compose/docker-compose.cluster.perf-round1-linux.override.yml b/.docker/compose/docker-compose.cluster.perf-round1-linux.override.yml new file mode 100644 index 000000000..22301b527 --- /dev/null +++ b/.docker/compose/docker-compose.cluster.perf-round1-linux.override.yml @@ -0,0 +1,56 @@ +services: + node1: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=48 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=8 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=16 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=12 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824 + - RUSTFS_RUNTIME_WORKER_THREADS=12 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=512 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=40 + + node2: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=48 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=8 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=16 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=12 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824 + - RUSTFS_RUNTIME_WORKER_THREADS=12 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=512 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=40 + + node3: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=48 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=8 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=16 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=12 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824 + - RUSTFS_RUNTIME_WORKER_THREADS=12 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=512 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=40 + + node4: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=48 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=8 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=16 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=12 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824 + - RUSTFS_RUNTIME_WORKER_THREADS=12 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=512 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=40 diff --git a/.docker/compose/docker-compose.cluster.perf-round1.override.yml b/.docker/compose/docker-compose.cluster.perf-round1.override.yml new file mode 100644 index 000000000..d5fd691d0 --- /dev/null +++ b/.docker/compose/docker-compose.cluster.perf-round1.override.yml @@ -0,0 +1,56 @@ +services: + node1: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=32 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=8 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=16777216 + - RUSTFS_RUNTIME_WORKER_THREADS=6 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=256 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=30 + + node2: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=32 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=8 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=16777216 + - RUSTFS_RUNTIME_WORKER_THREADS=6 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=256 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=30 + + node3: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=32 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=8 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=16777216 + - RUSTFS_RUNTIME_WORKER_THREADS=6 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=256 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=30 + + node4: + environment: + - RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=32 + - RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 + - RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6 + - RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12 + - RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=8 + - RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + - RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=16777216 + - RUSTFS_RUNTIME_WORKER_THREADS=6 + - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=256 + - RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + - RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e9ef5524..90d39b1ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -204,7 +204,7 @@ jobs: with: tool: s3s-e2e git: https://github.com/s3s-project/s3s.git - rev: 4a04a670cf41274d9be9ab65dc36f4aa3f92fbad + rev: 62cb4a71dd759a6ec56b64c4c42fcc183a2c6a52 - name: Run end-to-end tests run: | diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 6176cb314..8f0b15152 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -86,7 +86,7 @@ jobs: - name: Build with profiling optimizations run: | - RUSTFLAGS="-C force-frame-pointers=yes -C debug-assertions=off" \ + RUSTFLAGS="-C force-frame-pointers=yes -C debug-assertions=off --cfg tokio_unstable" \ cargo +nightly build --profile profiling -p rustfs --bins - name: Run performance profiling diff --git a/Cargo.lock b/Cargo.lock index 9d3171ee9..1c4331f5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -583,9 +583,9 @@ dependencies = [ [[package]] name = "asn1-rs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -622,9 +622,9 @@ dependencies = [ [[package]] name = "astral-tokio-tar" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ce73b17c62717c4b6a9af10b43e87c578b0cac27e00666d48304d3b7d2c0693" +checksum = "cb50a7aae84a03bf55b067832bc376f4961b790c97e64d3eacee97d389b90277" dependencies = [ "filetime", "futures-core", @@ -2601,9 +2601,9 @@ checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -3532,9 +3532,9 @@ dependencies = [ [[package]] name = "dial9-tokio-telemetry" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bc5e50c103d4916a4a1b95784d9cc515d96bb2d093c8ea37c2cc0d85d92dd" +checksum = "45521f70d67ff82b19ffddb55c7f039ee24c44a172aeb03ec2142f1808ae8562" dependencies = [ "arc-swap", "bon", @@ -3692,7 +3692,7 @@ dependencies = [ "md5 0.8.0", "rand 0.10.1", "rcgen", - "reqwest 0.13.3", + "reqwest", "rmp-serde", "russh", "russh-sftp", @@ -3916,9 +3916,9 @@ dependencies = [ [[package]] name = "enumset" -version = "1.1.12" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96a4a12fe60ac746ae295a1a4ecb5bb02debc20856506c8635288065f142de" +checksum = "839c4174b41e75c8f7306110b2c51996a293b8d1d850edd529011841d9fede7d" dependencies = [ "enumset_derive", ] @@ -4444,7 +4444,7 @@ dependencies = [ "hmac 0.13.0", "http 1.4.0", "jsonwebtoken", - "reqwest 0.13.3", + "reqwest", "rustc_version", "rustls", "rustls-pki-types", @@ -4495,14 +4495,14 @@ dependencies = [ "http-body-util", "hyper", "lazy_static", - "opentelemetry", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk", + "opentelemetry 0.31.0", + "opentelemetry-semantic-conventions 0.31.0", + "opentelemetry_sdk 0.31.0", "percent-encoding", "pin-project", "prost 0.14.3", "prost-types 0.14.3", - "reqwest 0.13.3", + "reqwest", "rustc_version", "serde", "serde_json", @@ -4513,7 +4513,7 @@ dependencies = [ "tonic-prost", "tower", "tracing", - "tracing-opentelemetry", + "tracing-opentelemetry 0.32.1", ] [[package]] @@ -5827,9 +5827,9 @@ dependencies = [ [[package]] name = "libbz2-rs-sys" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fc329e1457d97a9d58a4e2ca49e3be572431a7e096008efc2e3a3c19d428f4" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" @@ -6371,13 +6371,14 @@ dependencies = [ [[package]] name = "ml-kem" -version = "0.3.0-rc.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8198b5db27ac9773534c371751a59dc18aec8b80aa141e69abfdd1dec2e3f78c" +checksum = "5e15f3e5b957493873e396a66914e83e616b6afe335cdef7efe5c6e1216aba66" dependencies = [ "hybrid-array", "kem", "module-lattice", + "pkcs8 0.11.0", "rand_core 0.10.1", "sha3", ] @@ -6415,9 +6416,9 @@ dependencies = [ [[package]] name = "mqttbytes-core-next" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2558e8a9dd1e9ad2e771384ed983216741f60e21e5974b43b41f9980bff3e3d7" +checksum = "199b7cddf2d1bba6248d9dbaf1e58c91cd3ab05f52559b566c9d75ff303f887b" dependencies = [ "bytes", "thiserror 2.0.18", @@ -6744,9 +6745,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-format" @@ -7066,61 +7067,71 @@ dependencies = [ ] [[package]] -name = "opentelemetry-appender-tracing" -version = "0.31.1" +name = "opentelemetry" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ - "opentelemetry", - "tracing", - "tracing-core", - "tracing-log", - "tracing-opentelemetry", - "tracing-subscriber", -] - -[[package]] -name = "opentelemetry-http" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" -dependencies = [ - "async-trait", - "bytes", - "http 1.4.0", - "opentelemetry", - "reqwest 0.12.28", -] - -[[package]] -name = "opentelemetry-otlp" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" -dependencies = [ - "flate2", - "http 1.4.0", - "opentelemetry", - "opentelemetry-http", - "opentelemetry-proto", - "opentelemetry_sdk", - "prost 0.14.3", - "reqwest 0.12.28", + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", "thiserror 2.0.18", "tracing", ] [[package]] -name = "opentelemetry-proto" -version = "0.31.0" +name = "opentelemetry-appender-tracing" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" dependencies = [ - "opentelemetry", - "opentelemetry_sdk", + "opentelemetry 0.32.0", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + +[[package]] +name = "opentelemetry-http" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.0", + "opentelemetry 0.32.0", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" +dependencies = [ + "flate2", + "http 1.4.0", + "opentelemetry 0.32.0", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk 0.32.0", + "prost 0.14.3", + "reqwest", + "thiserror 2.0.18", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "opentelemetry 0.32.0", + "opentelemetry_sdk 0.32.0", "prost 0.14.3", - "tonic", - "tonic-prost", ] [[package]] @@ -7130,14 +7141,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" [[package]] -name = "opentelemetry-stdout" -version = "0.31.0" +name = "opentelemetry-semantic-conventions" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc8887887e169414f637b18751487cce4e095be787d23fad13c454e2fb1b3811" +checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" + +[[package]] +name = "opentelemetry-stdout" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b1c6a247d79091f0062a5f4bd058589525cf987a8d4c169440d9c1be72f0ad" dependencies = [ "chrono", - "opentelemetry", - "opentelemetry_sdk", + "opentelemetry 0.32.0", + "opentelemetry_sdk 0.32.0", ] [[package]] @@ -7149,10 +7166,26 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry", + "opentelemetry 0.31.0", "percent-encoding", "rand 0.9.4", "thiserror 2.0.18", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368afaed344110f40b179bb8fbe54bc52d98f9bd2b281799ef32487c2650c956" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.32.0", + "percent-encoding", + "portable-atomic", + "rand 0.9.4", + "thiserror 2.0.18", "tokio", "tokio-stream", ] @@ -7299,7 +7332,7 @@ dependencies = [ [[package]] name = "pageant" version = "0.2.0" -source = "git+https://github.com/simon-escapecode/russh?rev=5cac2ed84945f9b80a52b673e058f2032bbe98ec#5cac2ed84945f9b80a52b673e058f2032bbe98ec" +source = "git+https://github.com/Eugeny/russh?rev=fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b#fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" dependencies = [ "byteorder", "bytes", @@ -8311,7 +8344,7 @@ dependencies = [ "names", "pprof-pyroscope-fork", "prost 0.14.3", - "reqwest 0.13.3", + "reqwest", "serde_json", "thiserror 2.0.18", "url", @@ -8813,46 +8846,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "reqwest" version = "0.13.3" @@ -9025,9 +9018,9 @@ dependencies = [ [[package]] name = "rumqttc-core-next" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8a93655cd2d9d3faf7e039caa4eb39d9f4b49c336ade913c92655031c9fca71" +checksum = "660f3b3e2dc9e138e53c4db7f0ec42fe3386c751ccdff2fbd1d075a4149254cd" dependencies = [ "async-tungstenite", "futures-io", @@ -9043,18 +9036,18 @@ dependencies = [ [[package]] name = "rumqttc-next" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6cd37028e449ffb206a1ddc60086a1a5bac578b45be91b487dccb47a930334" +checksum = "e101c834b5ad0c5c82c5bcc7ac0b0dca70baba92fe7b903f441b87bef1f45387" dependencies = [ "rumqttc-v5-next", ] [[package]] name = "rumqttc-v5-next" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796245ad9ebd2b301ed20263ccf0ee430b6bc511b3ece7f815288a72d393e763" +checksum = "d965c65b95e12ed1cb564c2fa5fd70e3f3885b812f7429de6e5ff22f090e27f1" dependencies = [ "async-tungstenite", "bytes", @@ -9077,8 +9070,8 @@ dependencies = [ [[package]] name = "russh" -version = "0.60.2" -source = "git+https://github.com/simon-escapecode/russh?rev=5cac2ed84945f9b80a52b673e058f2032bbe98ec#5cac2ed84945f9b80a52b673e058f2032bbe98ec" +version = "0.60.3" +source = "git+https://github.com/Eugeny/russh?rev=fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b#fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" dependencies = [ "aead 0.6.0-rc.10", "aes 0.8.4", @@ -9158,8 +9151,8 @@ dependencies = [ [[package]] name = "russh-cryptovec" -version = "0.59.0" -source = "git+https://github.com/simon-escapecode/russh?rev=5cac2ed84945f9b80a52b673e058f2032bbe98ec#5cac2ed84945f9b80a52b673e058f2032bbe98ec" +version = "0.60.3" +source = "git+https://github.com/Eugeny/russh?rev=fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b#fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" dependencies = [ "log", "nix 0.31.3", @@ -9188,7 +9181,7 @@ dependencies = [ [[package]] name = "russh-util" version = "0.52.0" -source = "git+https://github.com/simon-escapecode/russh?rev=5cac2ed84945f9b80a52b673e058f2032bbe98ec#5cac2ed84945f9b80a52b673e058f2032bbe98ec" +source = "git+https://github.com/Eugeny/russh?rev=fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b#fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" dependencies = [ "chrono", "tokio", @@ -9312,13 +9305,13 @@ dependencies = [ "metrics", "mimalloc", "mime_guess", - "opentelemetry", - "opentelemetry_sdk", + "opentelemetry 0.32.0", + "opentelemetry_sdk 0.32.0", "percent-encoding", "pin-project-lite", "pprof-pyroscope-fork", "rand 0.10.1", - "reqwest 0.13.3", + "reqwest", "rmp-serde", "rsa 0.10.0-rc.18", "rust-embed", @@ -9380,7 +9373,7 @@ dependencies = [ "tower", "tower-http", "tracing", - "tracing-opentelemetry", + "tracing-opentelemetry 0.33.0", "tracing-subscriber", "url", "urlencoding", @@ -9544,8 +9537,8 @@ dependencies = [ "memmap2 0.9.10", "metrics", "num_cpus", - "opentelemetry", - "opentelemetry_sdk", + "opentelemetry 0.32.0", + "opentelemetry_sdk 0.32.0", "parking_lot 0.12.5", "path-absolutize", "pin-project-lite", @@ -9555,7 +9548,7 @@ dependencies = [ "reed-solomon-erasure", "reed-solomon-simd", "regex", - "reqwest 0.13.3", + "reqwest", "rmp", "rmp-serde", "rustfs-checksums", @@ -9594,7 +9587,7 @@ dependencies = [ "tonic", "tower", "tracing", - "tracing-opentelemetry", + "tracing-opentelemetry 0.33.0", "tracing-subscriber", "url", "urlencoding", @@ -9668,7 +9661,7 @@ dependencies = [ "openidconnect", "pollster", "rand 0.10.1", - "reqwest 0.13.3", + "reqwest", "rustfs-config", "rustfs-credentials", "rustfs-crypto", @@ -9772,7 +9765,7 @@ dependencies = [ "http-body-util", "hyper", "moka", - "reqwest 0.13.3", + "reqwest", "rustfs-credentials", "rustfs-policy", "rustfs-utils", @@ -9799,7 +9792,7 @@ dependencies = [ "md5 0.8.0", "moka", "rand 0.10.1", - "reqwest 0.13.3", + "reqwest", "rustfs-utils", "serde", "serde_json", @@ -9857,6 +9850,7 @@ dependencies = [ "async-trait", "axum", "chrono", + "criterion", "form_urlencoded", "hashbrown 0.17.1", "metrics", @@ -9915,12 +9909,12 @@ dependencies = [ "metrics", "num_cpus", "nvml-wrapper", - "opentelemetry", + "opentelemetry 0.32.0", "opentelemetry-appender-tracing", "opentelemetry-otlp", - "opentelemetry-semantic-conventions", + "opentelemetry-semantic-conventions 0.32.0", "opentelemetry-stdout", - "opentelemetry_sdk", + "opentelemetry_sdk 0.32.0", "percent-encoding", "pyroscope", "rustfs-audit", @@ -9941,7 +9935,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-error", - "tracing-opentelemetry", + "tracing-opentelemetry 0.33.0", "tracing-subscriber", "zstd", ] @@ -9959,7 +9953,7 @@ dependencies = [ "moka", "pollster", "regex", - "reqwest 0.13.3", + "reqwest", "rustfs-config", "rustfs-credentials", "rustfs-crypto", @@ -10064,7 +10058,7 @@ dependencies = [ "md-5 0.11.0", "pin-project-lite", "rand 0.10.1", - "reqwest 0.13.3", + "reqwest", "rustfs-common", "rustfs-config", "rustfs-utils", @@ -10196,7 +10190,7 @@ dependencies = [ "parking_lot 0.12.5", "pulsar", "redis", - "reqwest 0.13.3", + "reqwest", "rumqttc-next", "rustfs-config", "rustfs-ecstore", @@ -10232,7 +10226,7 @@ dependencies = [ "metrics", "moka", "regex", - "reqwest 0.13.3", + "reqwest", "rustfs-config", "rustfs-utils", "serde", @@ -10327,7 +10321,7 @@ dependencies = [ "async-trait", "bytes", "http 1.4.0", - "reqwest 0.13.3", + "reqwest", "rustify_derive", "serde", "serde_json", @@ -11555,9 +11549,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4deba334e1190ba7cb498327affa11e5ece10d26a30ab2f27fcf09504b8d8b6" +checksum = "14311e7e9a03114cd4b65eedd54e8fed2945e17f08586ae97ef53bc0669f9581" dependencies = [ "libc", "memchr", @@ -12126,9 +12120,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "bitflags 2.11.1", @@ -12235,7 +12229,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" dependencies = [ "js-sys", - "opentelemetry", + "opentelemetry 0.31.0", + "tracing", + "tracing-core", + "tracing-subscriber", + "web-time", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" +dependencies = [ + "js-sys", + "opentelemetry 0.32.0", "smallvec", "tracing", "tracing-core", @@ -12501,7 +12509,7 @@ dependencies = [ "async-trait", "derive_builder", "http 1.4.0", - "reqwest 0.13.3", + "reqwest", "rustify", "rustify_derive", "serde", diff --git a/Cargo.toml b/Cargo.toml index 5cd3eb4be..d2c18d0c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -145,7 +145,7 @@ tonic = { version = "0.14.6", features = ["gzip"] } tonic-prost = { version = "0.14.6" } tonic-prost-build = { version = "0.14.6" } tower = { version = "0.5.3", features = ["timeout"] } -tower-http = { version = "0.6.10", features = ["cors"] } +tower-http = { version = "0.6.11", features = ["cors"] } # Serialization and Data Formats bytes = { version = "1.11.1", features = ["serde"] } @@ -154,7 +154,7 @@ byteorder = "1.5.0" flatbuffers = "25.12.19" form_urlencoded = "1.2.2" prost = "0.14.3" -quick-xml = "0.40.0" +quick-xml = "0.40.1" rmp = { version = "0.8.15" } rmp-serde = { version = "1.3.1" } serde = { version = "1.0.228", features = ["derive"] } @@ -194,7 +194,7 @@ tokio-postgres-rustls = "0.13" # Utilities and Tools anyhow = "1.0.102" arc-swap = "1.9.1" -astral-tokio-tar = "0.6.1" +astral-tokio-tar = "0.6.2" atoi = "2.0.0" atomic_enum = "0.3.0" aws-config = { version = "1.8.16" } @@ -216,7 +216,7 @@ crossbeam-deque = "0.8.6" crossbeam-utils = "0.8.21" datafusion = "53.1.0" derive_builder = "0.20.2" -enumset = "1.1.12" +enumset = "1.1.13" faster-hex = "0.10.0" flate2 = "1.1.9" glob = "0.3.3" @@ -254,7 +254,7 @@ rayon = "1.12.0" reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] } reed-solomon-simd = "3.1.0" regex = { version = "1.12.3" } -rumqttc = { package = "rumqttc-next", version = "0.33.0", features = ["websocket"] } +rumqttc = { package = "rumqttc-next", version = "0.33.1", features = ["websocket"] } redis = { version = "1.2.1", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] } rustix = { version = "1.1.4", features = ["fs"] } rust-embed = { version = "8.11.0" } @@ -269,7 +269,7 @@ snafu = "0.9.0" snap = "1.1.1" starshard = { version = "2.2.0", features = ["rayon", "async", "serde"] } strum = { version = "0.28.0", features = ["derive"] } -sysinfo = "0.39.1" +sysinfo = "0.39.2" temp-env = "0.3.6" tempfile = "3.27.0" test-case = "3.3.1" @@ -277,7 +277,7 @@ thiserror = "2.0.18" tracing = { version = "0.1.44" } tracing-appender = "0.2.5" tracing-error = "0.2.1" -tracing-opentelemetry = "0.32.1" +tracing-opentelemetry = { version = "0.33" } tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] } transform-stream = "0.3.1" url = "2.5.8" @@ -294,12 +294,12 @@ zstd = "0.13.3" # Observability and Metrics metrics = "0.24.6" dial9-tokio-telemetry = "0.3" -opentelemetry = { version = "0.31.0" } -opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] } -opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] } -opentelemetry_sdk = { version = "0.31.0" } -opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] } -opentelemetry-stdout = { version = "0.31.0" } +opentelemetry = { version = "0.32.0" } +opentelemetry-appender-tracing = { version = "0.32.0", features = ["experimental_span_attributes", "experimental_metadata_attributes"] } +opentelemetry-otlp = { version = "0.32.0", features = ["gzip-http", "reqwest-rustls"] } +opentelemetry_sdk = { version = "0.32.0", features = ["rt-tokio"] } +opentelemetry-semantic-conventions = { version = "0.32.0", features = ["semconv_experimental"] } +opentelemetry-stdout = { version = "0.32.0" } pyroscope = { version = "2.0.3", features = ["backend-pprof-rs"] } # FTP and SFTP @@ -307,7 +307,7 @@ libunftp = { version = "0.23.0", features = ["experimental"] } unftp-core = "0.1.0" suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] } rcgen = "0.14.8" -russh = "0.60.2" +russh = { version = "0.60.3", git = "https://github.com/Eugeny/russh", rev = "fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" } russh-sftp = "2.1.2" # WebDAV @@ -329,12 +329,6 @@ pprof = { package = "pprof-pyroscope-fork", version = "0.1500.4", features = ["f [workspace.metadata.cargo-shear] ignored = ["rustfs"] -[patch.crates-io] -# Pinned to the simon-escapecode/russh fork carrying the upstream fix at -# https://github.com/Eugeny/russh/pull/702. Drops out when a russh release -# containing the fix is published. -russh = { git = "https://github.com/simon-escapecode/russh", rev = "5cac2ed84945f9b80a52b673e058f2032bbe98ec" } - [profile.release] opt-level = 3 diff --git a/Dockerfile.decommission-local b/Dockerfile.decommission-local index 1ccf3ce70..d3561baed 100644 --- a/Dockerfile.decommission-local +++ b/Dockerfile.decommission-local @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM rust:1.93-trixie +FROM rust:1.95-trixie RUN set -eux; \ export DEBIAN_FRONTEND=noninteractive; \ diff --git a/Dockerfile.source b/Dockerfile.source index 6e3af9ded..f86e6f95e 100644 --- a/Dockerfile.source +++ b/Dockerfile.source @@ -31,7 +31,7 @@ ARG TARGETARCH # ----------------------------- # Build stage # ----------------------------- -FROM rust:1.93-trixie AS builder +FROM rust:1.95-trixie AS builder # Re-declare args after FROM ARG TARGETPLATFORM diff --git a/crates/notify/Cargo.toml b/crates/notify/Cargo.toml index 131593804..85b86f6a9 100644 --- a/crates/notify/Cargo.toml +++ b/crates/notify/Cargo.toml @@ -62,9 +62,14 @@ axum = { workspace = true } rustfs-utils = { workspace = true, features = ["path"] } serde_json = { workspace = true } time = { workspace = true } +criterion = { workspace = true } [lints] workspace = true [lib] doctest = false + +[[bench]] +name = "snapshot_mode_scan" +harness = false diff --git a/crates/notify/benches/snapshot_mode_scan.rs b/crates/notify/benches/snapshot_mode_scan.rs new file mode 100644 index 000000000..22907f537 --- /dev/null +++ b/crates/notify/benches/snapshot_mode_scan.rs @@ -0,0 +1,62 @@ +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use rustfs_notify::rules::RulesMap; +use rustfs_targets::arn::TargetID; +use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode}; +use tokio::runtime::Runtime; + +fn build_rule_map(target: &TargetID) -> RulesMap { + let mut rules_map = RulesMap::new(); + rules_map.add_rule_config(&[rustfs_s3_common::EventName::ObjectCreatedPut], "*".to_string(), target.clone()); + rules_map +} + +async fn build_map(mode: SnapshotMode, bucket_count: usize) -> AsyncShardedHashMap { + let map = AsyncShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, mode); + let target = TargetID::new("bench-target".to_string(), "webhook".to_string()); + + for i in 0..bucket_count { + let bucket = format!("bucket-{i}"); + map.insert(bucket, build_rule_map(&target)).await; + } + map +} + +async fn scan_target_bound(map: &AsyncShardedHashMap, target_id: &TargetID) -> bool { + let items = map.iter().await; + for (_bucket, rules_map) in items { + if rules_map.contains_target_id(target_id) { + return true; + } + } + false +} + +fn bench_snapshot_mode_scan(c: &mut Criterion) { + let rt = Runtime::new().expect("tokio runtime"); + + let mut group = c.benchmark_group("notify_rule_engine_scan_snapshot_mode"); + // Simulate medium-large config sets where full-map scan cost matters. + let bucket_sizes = [1_000usize, 10_000usize]; + + for bucket_count in bucket_sizes { + for (mode, mode_name) in [(SnapshotMode::Clone, "clone"), (SnapshotMode::Cached, "cached")] { + let map = rt.block_on(build_map(mode, bucket_count)); + let miss_target = TargetID::new("missing-target".to_string(), "webhook".to_string()); + + group.throughput(Throughput::Elements(bucket_count as u64)); + group.bench_with_input(BenchmarkId::new(mode_name, bucket_count), &bucket_count, |b, _| { + b.iter(|| { + let found = rt.block_on(async { + scan_target_bound(std::hint::black_box(&map), std::hint::black_box(&miss_target)).await + }); + std::hint::black_box(found); + }); + }); + } + } + + group.finish(); +} + +criterion_group!(benches, bench_snapshot_mode_scan); +criterion_main!(benches); diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 60ae3e9a2..a0bcbd9ea 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -51,11 +51,11 @@ crossbeam-deque = { workspace = true } crossbeam-utils = { workspace = true } num_cpus = { workspace = true } opentelemetry = { workspace = true } -opentelemetry-appender-tracing = { workspace = true, features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] } -opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] } +opentelemetry-appender-tracing = { workspace = true } +opentelemetry_sdk = { workspace = true } opentelemetry-stdout = { workspace = true } opentelemetry-otlp = { workspace = true } -opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } +opentelemetry-semantic-conventions = { workspace = true } percent-encoding = { workspace = true } serde = { workspace = true } tracing = { workspace = true, features = ["std", "attributes"] } diff --git a/deny.toml b/deny.toml index 8708c7498..1b0f1f316 100644 --- a/deny.toml +++ b/deny.toml @@ -45,6 +45,9 @@ allow-git = [ # Custom S3 server library with minio compatibility patches not yet upstreamed. # Pinned to a specific commit in workspace Cargo.toml. "https://github.com/rustfs/s3s", + # Temporary git source for russh until required upstream fixes are released. + # owner: rustfs-maintainers review: 2026-07 + "https://github.com/Eugeny/russh", ] [bans] diff --git a/docker-compose-simple.yml b/docker-compose-simple.yml index a39283fba..bdf06d83a 100644 --- a/docker-compose-simple.yml +++ b/docker-compose-simple.yml @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3.9" - services: # RustFS main service rustfs: diff --git a/docker-compose.decommission.yml b/docker-compose.decommission.yml index f1c34cef1..d8fc2b281 100644 --- a/docker-compose.decommission.yml +++ b/docker-compose.decommission.yml @@ -1,3 +1,17 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + services: rustfs-decommission-latest: image: ${RUSTFS_DOCKER_IMAGE:-rustfs-local:decommission-latest}