diff --git a/Cargo.lock b/Cargo.lock index 8af2498c4..d227106ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -682,9 +682,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.43" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" +checksum = "515a1f282e33d55983c499d7e9e87082e81cbc32974825bf9032f928392d5844" dependencies = [ "compression-codecs", "compression-core", @@ -1039,7 +1039,7 @@ dependencies = [ "http 0.2.12", "http 1.5.0", "http-body 1.1.0", - "lru 0.18.3", + "lru 0.18.4", "percent-encoding", "regex-lite", "sha2 0.11.0", @@ -2189,9 +2189,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +checksum = "2fe67f2944eef52fc7b106b8c9450d243a88701a0c065f7f57235e76abaed7df" dependencies = [ "brotli 8.0.4", "bzip2", @@ -2206,9 +2206,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" [[package]] name = "concurrent-queue" @@ -6248,9 +6248,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.18.3" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" dependencies = [ "hashbrown 0.17.1", ] @@ -6699,7 +6699,7 @@ dependencies = [ "futures-sink", "futures-util", "keyed_priority_queue", - "lru 0.18.3", + "lru 0.18.4", "mysql_common", "percent-encoding", "rand 0.10.2", @@ -10194,18 +10194,18 @@ dependencies = [ [[package]] name = "rustfs-mimalloc" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed388b8a3d55818c32973ded41df5215d33ee0f574d9ca03f2731b3007c3284b" +checksum = "85d1a75bd188260754c4fa8ace1bb1f6a9956ee36c1d330f98fed5d25e39fa15" dependencies = [ "rustfs-mimalloc-sys", ] [[package]] name = "rustfs-mimalloc-sys" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c507265df958f6b63f8ccdd21819374186bab5dbdcc3f9c5cf8bff4b2e3f5d0" +checksum = "5d3adbf24cbe37f040f856c04f8d4baf75626eacd4e8b17eb6b3d79de7fde692" dependencies = [ "cc", ] @@ -12299,7 +12299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index 271279430..20cc7048e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -145,7 +145,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" } async-channel = "2.5.0" async_zip = { default-features = false, version = "0.0.19" } mysql_async = { default-features = false, version = "0.37.1" } -async-compression = { version = "0.4.43" } +async-compression = { version = "0.4.44" } async-recursion = "1.1.1" async-trait = "0.1.92" async-nats = { version = "0.50.0", default-features = false } @@ -298,7 +298,7 @@ pretty_assertions = "1.4.1" rand = { version = "0.10.2" } ratelimit = "2.0.0" rayon = "1.12.0" -reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" } +rustfs-erasure-codec = { version = "8.0.2" } reed-solomon-simd = "3.1.0" regex = { version = "1.13.1" } rumqttc = { package = "rumqttc-next", version = "0.34.0" } @@ -366,7 +366,7 @@ russh-sftp = "2.4.0" dav-server = "0.11.0" # Performance Analysis and Memory Profiling -rustfs-mimalloc = { version = "0.5.1" } +rustfs-mimalloc = { version = "0.5.2" } hotpath = { version = "0.24.0", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.48" } diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index dc275bc81..9358e1a54 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -163,7 +163,7 @@ http-body = { workspace = true } http-body-util.workspace = true url.workspace = true uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] } -reed-solomon-erasure = { workspace = true, features = ["simd-accel"] } +rustfs-erasure-codec = { workspace = true, features = ["simd-accel"] } reed-solomon-simd = { workspace = true } lazy_static.workspace = true moka = { workspace = true, features = ["future", "sync"] } diff --git a/crates/ecstore/benches/erasure_benchmark.rs b/crates/ecstore/benches/erasure_benchmark.rs index a2f7512f7..c9f7adb52 100644 --- a/crates/ecstore/benches/erasure_benchmark.rs +++ b/crates/ecstore/benches/erasure_benchmark.rs @@ -122,10 +122,10 @@ fn bench_encode_performance(c: &mut Criterion) { }); group.finish(); - // Test direct reed-solomon-erasure implementation for large shards (>= 512 bytes) + // Test direct rustfs-erasure-codec implementation for large shards (>= 512 bytes) let shard_size = calc_shard_size(config.data_size, config.data_shards); if shard_size >= 512 && config.parity_shards > 0 { - use reed_solomon_erasure::galois_8::ReedSolomon; + use rustfs_erasure_codec::galois_8::ReedSolomon; let mut rse_group = c.benchmark_group("encode_rse_direct"); rse_group.throughput(Throughput::Bytes(config.data_size as u64)); @@ -204,10 +204,10 @@ fn bench_decode_performance(c: &mut Criterion) { ); group.finish(); - // Test direct reed-solomon-erasure decoding for large shards + // Test direct rustfs-erasure-codec decoding for large shards let shard_size = calc_shard_size(config.data_size, config.data_shards); if shard_size >= 512 && config.parity_shards > 0 { - use reed_solomon_erasure::galois_8::ReedSolomon; + use rustfs_erasure_codec::galois_8::ReedSolomon; if let Ok(rs) = ReedSolomon::new(config.data_shards, config.parity_shards) { let mut rse_group = c.benchmark_group("decode_rse_direct"); diff --git a/crates/ecstore/src/disk/error.rs b/crates/ecstore/src/disk/error.rs index c4c068800..7747cfa6d 100644 --- a/crates/ecstore/src/disk/error.rs +++ b/crates/ecstore/src/disk/error.rs @@ -845,7 +845,7 @@ mod tests { use std::error::Error as _; let error = DiskError::from(ErasureConstructionError::ModernEncoder { - source: reed_solomon_erasure::Error::TooManyShards, + source: rustfs_erasure_codec::Error::TooManyShards, }); let io_source = error.source().expect("DiskError::Io must expose its io::Error source"); assert!(io_source.is::()); @@ -856,7 +856,7 @@ mod tests { let encoder_source = construction_source .source() .expect("construction error must expose the encoder error"); - assert!(encoder_source.is::()); + assert!(encoder_source.is::()); } #[test] diff --git a/crates/ecstore/src/erasure/codec/bridge.rs b/crates/ecstore/src/erasure/codec/bridge.rs index 75388ac87..aa2c79e25 100644 --- a/crates/ecstore/src/erasure/codec/bridge.rs +++ b/crates/ecstore/src/erasure/codec/bridge.rs @@ -14,7 +14,7 @@ use crate::erasure::codec::workspace::RustfsCodecDecodeWorkspace; use crate::erasure::coding::Erasure; -use reed_solomon_erasure::galois_8::ReedSolomon; +use rustfs_erasure_codec::galois_8::ReedSolomon; use std::io; use std::sync::{Arc, OnceLock}; diff --git a/crates/ecstore/src/erasure/coding/erasure.rs b/crates/ecstore/src/erasure/coding/erasure.rs index 602b24789..682ca0cf9 100644 --- a/crates/ecstore/src/erasure/coding/erasure.rs +++ b/crates/ecstore/src/erasure/coding/erasure.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Erasure coding implementation using reed-solomon-erasure (GF(2^8)). +//! Erasure coding implementation using rustfs-erasure-codec (GF(2^8)). //! Supports legacy (reed-solomon-simd) for reading/healing old-version files. //! use bytes::{Bytes, BytesMut}; -use reed_solomon_erasure::galois_8::ReedSolomon; use reed_solomon_simd; +use rustfs_erasure_codec::galois_8::ReedSolomon; use smallvec::SmallVec; use std::{ collections::HashMap, @@ -69,7 +69,7 @@ impl EncodedBlock { } } -const MODERN_MAX_TOTAL_SHARDS: usize = ::ORDER; +const MODERN_MAX_TOTAL_SHARDS: usize = ::ORDER; const MODERN_REED_SOLOMON_CACHE_MAX_ENTRIES: usize = 64; const LEGACY_REED_SOLOMON_CACHE_MAX_ENTRIES: usize = 16; // Vec growth may retain twice the requested logical length. Keeping the logical @@ -109,7 +109,7 @@ pub enum ErasureConstructionError { #[error("failed to construct modern Reed-Solomon encoder")] ModernEncoder { #[source] - source: reed_solomon_erasure::Error, + source: rustfs_erasure_codec::Error, }, /// The legacy encoder wrapper failed to initialize. @@ -354,7 +354,7 @@ impl LegacyReedSolomonEncoder { } } -/// Reed-Solomon encoder using reed-solomon-erasure +/// Reed-Solomon encoder using rustfs-erasure-codec pub struct ReedSolomonEncoder { data_shards: usize, parity_shards: usize, @@ -372,7 +372,7 @@ impl Clone for ReedSolomonEncoder { } impl ReedSolomonEncoder { - fn try_new_typed(data_shards: usize, parity_shards: usize) -> Result { + fn try_new_typed(data_shards: usize, parity_shards: usize) -> Result { let encoder = if parity_shards > 0 { Some(cached_modern_reed_solomon(data_shards, parity_shards)?) } else { @@ -445,7 +445,7 @@ impl ReedSolomonEncoder { } } -fn cached_modern_reed_solomon(data_shards: usize, parity_shards: usize) -> Result, reed_solomon_erasure::Error> { +fn cached_modern_reed_solomon(data_shards: usize, parity_shards: usize) -> Result, rustfs_erasure_codec::Error> { let key = (data_shards, parity_shards); let cache = MODERN_REED_SOLOMON_CACHE.get_or_init(|| RwLock::new(HashMap::new())); @@ -1407,11 +1407,11 @@ mod tests { #[test] fn construction_errors_preserve_encoder_sources() { let modern = ErasureConstructionError::ModernEncoder { - source: reed_solomon_erasure::Error::TooManyShards, + source: rustfs_erasure_codec::Error::TooManyShards, }; assert!( std::error::Error::source(&modern) - .and_then(|source| source.downcast_ref::()) + .and_then(|source| source.downcast_ref::()) .is_some() ); diff --git a/crates/ecstore/src/error/mod.rs b/crates/ecstore/src/error/mod.rs index 52018d6b4..918774c59 100644 --- a/crates/ecstore/src/error/mod.rs +++ b/crates/ecstore/src/error/mod.rs @@ -1221,7 +1221,7 @@ mod tests { use std::error::Error as _; let error = StorageError::from(ErasureConstructionError::ModernEncoder { - source: reed_solomon_erasure::Error::TooManyShards, + source: rustfs_erasure_codec::Error::TooManyShards, }); let io_source = error.source().expect("StorageError::Io must expose its io::Error source"); assert!(io_source.is::()); @@ -1232,7 +1232,7 @@ mod tests { let encoder_source = construction_source .source() .expect("construction error must expose the encoder error"); - assert!(encoder_source.is::()); + assert!(encoder_source.is::()); } // The lifecycle transition worker relies on this arm alone to suppress the diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 20df64458..51aef9ef5 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -20,8 +20,8 @@ use std::alloc::{GlobalAlloc, Layout}; struct MiMallocAllocator; #[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))] -// SAFETY: allocation operations are forwarded unchanged to MiMalloc, so -// MiMalloc's GlobalAlloc guarantees apply to every returned pointer and layout. +// SAFETY: allocation operations are forwarded to MiMalloc with the +// corresponding GlobalAlloc size and alignment contracts. #[allow(unsafe_code)] unsafe impl GlobalAlloc for MiMallocAllocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { @@ -35,8 +35,8 @@ unsafe impl GlobalAlloc for MiMallocAllocator { } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - // SAFETY: ptr and layout came from this allocator and are forwarded unchanged. - unsafe { rustfs_mimalloc::MiMalloc.dealloc(ptr, layout) } + // SAFETY: ptr came from this allocator and layout.size() is the original allocation size. + unsafe { rustfs_mimalloc::MiMalloc::free_csize(ptr, layout.size()) } } unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {