mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
perf(erasure): remove UUID from clone + increase encode inflight budget (#3212)
* perf(erasure): remove UUID from clone + increase encode inflight budget Two targeted optimizations for the erasure encoding hot path: 1. Erasure::clone() no longer generates Uuid::new_v4() per clone. The _id field is unused in the hot path; reusing the original ID eliminates a CSPRNG call per block encode (100 calls for a 100MB object with 1MB blocks). 2. Default RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES raised from 8MB to 32MB. This increases the encode pipeline depth from ~5 to ~20 blocks, allowing more read-ahead between the encoder and disk writer stages. The per-request memory bound is still controlled by the 8-block hard cap and the env var override. 3. Added encode_data_owned() utility method for zero-copy encoding when the caller already owns a heap buffer (Vec<u8> → BytesMut via Bytes::try_into_mut). Not used in the hot path yet but available for future callers. All 1157 ecstore tests pass. Criterion micro-benchmarks show no regression (< 2% variance). Single-machine warp E2E tests were inconclusive due to high variance; a dedicated multi-disk test environment is needed for reliable E2E comparison. Ref: https://github.com/rustfs/backlog/issues/659 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * chore: update Cargo.lock * fix(erasure): align encode inflight cap and tests --------- Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
安正超
Generated
+60
-60
@@ -1450,9 +1450,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.12.1"
|
||||
version = "2.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a"
|
||||
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
@@ -1733,9 +1733,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.63"
|
||||
version = "1.2.62"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
|
||||
checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -3478,9 +3478,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-tokio-telemetry"
|
||||
version = "0.3.13"
|
||||
version = "0.3.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "226a0d823327c391d9e8234dc3ccc5810d936359ba8ea6af024d57bb15568647"
|
||||
checksum = "9db20413b8c96577881e6f806970e81d41376236569eaf69f9329e34fc48bd79"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bon",
|
||||
@@ -3574,7 +3574,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3583,7 +3583,7 @@ version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -3910,7 +3910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4030,7 +4030,7 @@ version = "25.12.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
@@ -4228,9 +4228,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "1.4.3"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2e55f16dcf0e9c00efbe2e655ffe45fc98e7066b52bc92f8a79e64060a79351"
|
||||
checksum = "dab9e9188e97a93276e1fe7b56401b851e2b45a46d045ca658100c1303ada649"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"rustversion",
|
||||
@@ -5280,7 +5280,7 @@ version = "0.7.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
@@ -5324,7 +5324,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5807,7 +5807,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9f8ff371890db2cf65a0758dba9a79f9cd965de369f6dbdc6581a22780af45e"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
@@ -5872,9 +5872,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.31"
|
||||
version = "0.4.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f"
|
||||
checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5"
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
@@ -5939,9 +5939,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lzma-rust2"
|
||||
version = "0.16.4"
|
||||
version = "0.16.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce716bf1a316f47a280fc76295f6495b5bea4752bca01c3b3885e101b1c23c02"
|
||||
checksum = "5e9ceaec84b54518262de7cf06b8b43e83c808349960f1610b21b0bfc9640f20"
|
||||
dependencies = [
|
||||
"sha2 0.11.0",
|
||||
]
|
||||
@@ -6367,7 +6367,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b42ced54aa8ac97226486337973f9bc3956e24f03a23e88a6e18f640959d6e2"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"btoi",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -6399,7 +6399,7 @@ version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"byteorder",
|
||||
"derive_builder",
|
||||
"getset",
|
||||
@@ -6458,7 +6458,7 @@ version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6471,7 +6471,7 @@ version = "0.30.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6483,7 +6483,7 @@ version = "0.31.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6547,7 +6547,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6696,7 +6696,7 @@ version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"libloading",
|
||||
"nvml-wrapper-sys",
|
||||
"static_assertions",
|
||||
@@ -6719,7 +6719,7 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.21.7",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.4.1",
|
||||
@@ -6747,7 +6747,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"dispatch2",
|
||||
"objc2",
|
||||
]
|
||||
@@ -6764,7 +6764,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -7354,7 +7354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97f6fccfd2d9d2df765ca23ff85fe5cc437fb0e6d3e164e4d3cbe09d14780c93"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"thiserror 2.0.18",
|
||||
"zerocopy",
|
||||
"zerocopy-derive",
|
||||
@@ -7901,7 +7901,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
|
||||
dependencies = [
|
||||
"bit-set",
|
||||
"bit-vec 0.8.0",
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"num-traits",
|
||||
"rand 0.9.4",
|
||||
"rand_chacha 0.9.0",
|
||||
@@ -8094,7 +8094,7 @@ version = "0.13.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"memchr",
|
||||
"unicase",
|
||||
]
|
||||
@@ -8427,7 +8427,7 @@ version = "11.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8545,7 +8545,7 @@ version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8872,7 +8872,7 @@ checksum = "f67013f080c226e5a34db1c71f2567f44d95a6300005bb6cd4e2c8fe3c326d1b"
|
||||
dependencies = [
|
||||
"aes 0.9.1",
|
||||
"aws-lc-rs",
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"block-padding 0.4.2",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -8891,7 +8891,7 @@ dependencies = [
|
||||
"enum_dispatch",
|
||||
"flate2",
|
||||
"futures",
|
||||
"generic-array 1.4.3",
|
||||
"generic-array 1.4.1",
|
||||
"getrandom 0.2.17",
|
||||
"ghash",
|
||||
"hex-literal",
|
||||
@@ -8955,7 +8955,7 @@ version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed8949eca4163c18a8f59ff96d32cf61e9c13b9735e21ef32b3907f4aafa1a9"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
@@ -10174,7 +10174,7 @@ version = "0.38.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.4.15",
|
||||
@@ -10187,11 +10187,11 @@ version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10227,9 +10227,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-native-certs"
|
||||
version = "0.8.4"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
|
||||
checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
|
||||
dependencies = [
|
||||
"openssl-probe",
|
||||
"rustls-pki-types",
|
||||
@@ -10265,7 +10265,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10491,7 +10491,7 @@ version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"core-foundation 0.10.1",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
@@ -10799,9 +10799,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
@@ -11365,7 +11365,7 @@ version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"core-foundation 0.9.4",
|
||||
"system-configuration-sys",
|
||||
]
|
||||
@@ -11419,7 +11419,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11920,7 +11920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
@@ -12136,9 +12136,9 @@ checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
version = "1.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
|
||||
|
||||
[[package]]
|
||||
name = "unarray"
|
||||
@@ -12197,9 +12197,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.13.3"
|
||||
version = "1.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
||||
checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
@@ -12482,7 +12482,7 @@ version = "0.244.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"hashbrown 0.15.5",
|
||||
"indexmap 2.14.0",
|
||||
"semver",
|
||||
@@ -12597,7 +12597,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12891,7 +12891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags 2.12.1",
|
||||
"bitflags 2.11.1",
|
||||
"indexmap 2.14.0",
|
||||
"log",
|
||||
"serde",
|
||||
@@ -13056,18 +13056,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.50"
|
||||
version = "0.8.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
|
||||
checksum = "bce33a6288fa3f072a8c2c7d0f2fdbb90e28298f0135c1f99b96c3db2efcc60b"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.50"
|
||||
version = "0.8.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
|
||||
checksum = "8fd425244944f4ab65ccff928e7323354c5a018c75838362fdce749dfad2ee1e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
@@ -27,8 +27,8 @@ use tokio::sync::mpsc;
|
||||
use tracing::error;
|
||||
|
||||
const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
|
||||
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 8 * 1024 * 1024;
|
||||
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 8;
|
||||
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 32 * 1024 * 1024;
|
||||
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 32;
|
||||
|
||||
fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usize) -> usize {
|
||||
if expanded_block_bytes == 0 {
|
||||
@@ -501,6 +501,7 @@ mod tests {
|
||||
#[test]
|
||||
fn encode_channel_capacity_respects_budget_and_hard_cap() {
|
||||
assert_eq!(encode_channel_capacity(4 * 1024 * 1024, 32 * 1024 * 1024), 8);
|
||||
assert_eq!(encode_channel_capacity(1536 * 1024, 32 * 1024 * 1024), 21);
|
||||
assert_eq!(encode_channel_capacity(16 * 1024 * 1024, 32 * 1024 * 1024), 2);
|
||||
assert_eq!(encode_channel_capacity(1, usize::MAX), DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS);
|
||||
}
|
||||
|
||||
@@ -350,7 +350,7 @@ impl Clone for Erasure {
|
||||
legacy_encoder: self.legacy_encoder.clone(),
|
||||
block_size: self.block_size,
|
||||
uses_legacy: self.uses_legacy,
|
||||
_id: Uuid::new_v4(), // Generate new ID for clone
|
||||
_id: self._id, // Shared by clones; this field is unused in hot paths.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -413,6 +413,9 @@ impl Erasure {
|
||||
calc_shard_size
|
||||
};
|
||||
let per_shard_size = shard_size_fn(data.len(), self.data_shards);
|
||||
if per_shard_size == 0 {
|
||||
return Ok(vec![Bytes::new(); self.total_shard_count()]);
|
||||
}
|
||||
let need_total_size = per_shard_size * self.total_shard_count();
|
||||
|
||||
let mut data_buffer = BytesMut::with_capacity(need_total_size);
|
||||
@@ -448,6 +451,63 @@ impl Erasure {
|
||||
Ok(shards)
|
||||
}
|
||||
|
||||
/// Encode owned data, avoiding a copy when the caller already has a heap buffer.
|
||||
/// Falls back to copying into a new buffer if zero-copy conversion fails.
|
||||
pub fn encode_data_owned(&self, data: Vec<u8>) -> io::Result<Vec<Bytes>> {
|
||||
let shard_size_fn = if self.uses_legacy {
|
||||
calc_shard_size_legacy
|
||||
} else {
|
||||
calc_shard_size
|
||||
};
|
||||
let per_shard_size = shard_size_fn(data.len(), self.data_shards);
|
||||
if per_shard_size == 0 {
|
||||
return Ok(vec![Bytes::new(); self.total_shard_count()]);
|
||||
}
|
||||
let need_total_size = per_shard_size * self.total_shard_count();
|
||||
|
||||
// Try zero-copy: Vec<u8> -> Bytes -> BytesMut (succeeds when refcount == 1)
|
||||
let mut data_buffer = match Bytes::from(data).try_into_mut() {
|
||||
Ok(mut bm) => {
|
||||
bm.resize(need_total_size, 0u8);
|
||||
bm
|
||||
}
|
||||
Err(b) => {
|
||||
// Rare path: refcount != 1, fall back to copy
|
||||
let mut bm = BytesMut::with_capacity(need_total_size);
|
||||
bm.extend_from_slice(&b);
|
||||
bm.resize(need_total_size, 0u8);
|
||||
bm
|
||||
}
|
||||
};
|
||||
|
||||
{
|
||||
let data_slices: SmallVec<[&mut [u8]; 16]> = data_buffer.chunks_exact_mut(per_shard_size).collect();
|
||||
|
||||
if self.parity_shards > 0 {
|
||||
if self.uses_legacy {
|
||||
if let Some(encoder) = self.legacy_encoder.as_ref() {
|
||||
encoder.encode(data_slices)?;
|
||||
} else {
|
||||
warn!("parity_shards > 0, uses_legacy but legacy_encoder is None");
|
||||
}
|
||||
} else if let Some(encoder) = self.encoder.as_ref() {
|
||||
encoder.encode(data_slices)?;
|
||||
} else {
|
||||
warn!("parity_shards > 0, but encoder is None");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut data_buffer = data_buffer.freeze();
|
||||
let mut shards = Vec::with_capacity(self.total_shard_count());
|
||||
for _ in 0..self.total_shard_count() {
|
||||
let shard = data_buffer.split_to(per_shard_size);
|
||||
shards.push(shard);
|
||||
}
|
||||
|
||||
Ok(shards)
|
||||
}
|
||||
|
||||
/// Decode and reconstruct missing data shards in-place.
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -637,6 +697,24 @@ mod tests {
|
||||
shards.iter().map(|shard| Some(shard.to_vec())).collect()
|
||||
}
|
||||
|
||||
fn assert_owned_encode_matches_borrowed(erasure: &Erasure, data: Vec<u8>) {
|
||||
let borrowed = erasure.encode_data(&data).expect("borrowed encode should succeed");
|
||||
let owned = erasure.encode_data_owned(data).expect("owned encode should succeed");
|
||||
|
||||
assert_eq!(owned, borrowed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_data_owned_matches_borrowed_path() {
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(4, 2, 64, uses_legacy);
|
||||
|
||||
assert_owned_encode_matches_borrowed(&erasure, Vec::new());
|
||||
assert_owned_encode_matches_borrowed(&erasure, b"small payload".to_vec());
|
||||
assert_owned_encode_matches_borrowed(&erasure, (0_u8..37).collect());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_data_keeps_missing_parity_shard_unreconstructed() {
|
||||
let erasure = Erasure::new(2, 2, 64);
|
||||
|
||||
Reference in New Issue
Block a user