mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-17 18:27:49 +00:00
update ec share size
update bitrot
This commit is contained in:
@@ -93,6 +93,10 @@ pub struct ErasureInfo {
|
|||||||
pub checksums: Vec<ChecksumInfo>,
|
pub checksums: Vec<ChecksumInfo>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize {
|
||||||
|
(block_size.div_ceil(data_shards) + 1) & !1
|
||||||
|
}
|
||||||
|
|
||||||
impl ErasureInfo {
|
impl ErasureInfo {
|
||||||
pub fn get_checksum_info(&self, part_number: usize) -> ChecksumInfo {
|
pub fn get_checksum_info(&self, part_number: usize) -> ChecksumInfo {
|
||||||
for sum in &self.checksums {
|
for sum in &self.checksums {
|
||||||
@@ -109,7 +113,7 @@ impl ErasureInfo {
|
|||||||
|
|
||||||
/// Calculate the size of each shard.
|
/// Calculate the size of each shard.
|
||||||
pub fn shard_size(&self) -> usize {
|
pub fn shard_size(&self) -> usize {
|
||||||
self.block_size.div_ceil(self.data_blocks)
|
calc_shard_size(self.block_size, self.data_blocks)
|
||||||
}
|
}
|
||||||
/// Calculate the total erasure file size for a given original size.
|
/// Calculate the total erasure file size for a given original size.
|
||||||
// Returns the final erasure size from the original size
|
// Returns the final erasure size from the original size
|
||||||
@@ -120,7 +124,7 @@ impl ErasureInfo {
|
|||||||
|
|
||||||
let num_shards = total_length / self.block_size;
|
let num_shards = total_length / self.block_size;
|
||||||
let last_block_size = total_length % self.block_size;
|
let last_block_size = total_length % self.block_size;
|
||||||
let last_shard_size = last_block_size.div_ceil(self.data_blocks);
|
let last_shard_size = calc_shard_size(last_block_size, self.data_blocks);
|
||||||
num_shards * self.shard_size() + last_shard_size
|
num_shards * self.shard_size() + last_shard_size
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+26
-29
@@ -1,13 +1,12 @@
|
|||||||
use crate::{Reader, Writer};
|
|
||||||
use pin_project_lite::pin_project;
|
use pin_project_lite::pin_project;
|
||||||
use rustfs_utils::{HashAlgorithm, read_full, write_all};
|
use rustfs_utils::{HashAlgorithm, read_full, write_all};
|
||||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite};
|
||||||
|
|
||||||
pin_project! {
|
pin_project! {
|
||||||
/// BitrotReader reads (hash+data) blocks from an async reader and verifies hash integrity.
|
/// BitrotReader reads (hash+data) blocks from an async reader and verifies hash integrity.
|
||||||
pub struct BitrotReader {
|
pub struct BitrotReader<R> {
|
||||||
#[pin]
|
#[pin]
|
||||||
inner: Box<dyn Reader>,
|
inner: R,
|
||||||
hash_algo: HashAlgorithm,
|
hash_algo: HashAlgorithm,
|
||||||
shard_size: usize,
|
shard_size: usize,
|
||||||
buf: Vec<u8>,
|
buf: Vec<u8>,
|
||||||
@@ -19,14 +18,12 @@ pin_project! {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BitrotReader {
|
impl<R> BitrotReader<R>
|
||||||
/// Get a reference to the underlying reader.
|
where
|
||||||
pub fn get_ref(&self) -> &dyn Reader {
|
R: AsyncRead + Unpin + Send + Sync,
|
||||||
&*self.inner
|
{
|
||||||
}
|
|
||||||
|
|
||||||
/// Create a new BitrotReader.
|
/// Create a new BitrotReader.
|
||||||
pub fn new(inner: Box<dyn Reader>, shard_size: usize, algo: HashAlgorithm) -> Self {
|
pub fn new(inner: R, shard_size: usize, algo: HashAlgorithm) -> Self {
|
||||||
let hash_size = algo.size();
|
let hash_size = algo.size();
|
||||||
Self {
|
Self {
|
||||||
inner,
|
inner,
|
||||||
@@ -86,9 +83,9 @@ impl BitrotReader {
|
|||||||
|
|
||||||
pin_project! {
|
pin_project! {
|
||||||
/// BitrotWriter writes (hash+data) blocks to an async writer.
|
/// BitrotWriter writes (hash+data) blocks to an async writer.
|
||||||
pub struct BitrotWriter {
|
pub struct BitrotWriter<W> {
|
||||||
#[pin]
|
#[pin]
|
||||||
inner: Writer,
|
inner: W,
|
||||||
hash_algo: HashAlgorithm,
|
hash_algo: HashAlgorithm,
|
||||||
shard_size: usize,
|
shard_size: usize,
|
||||||
buf: Vec<u8>,
|
buf: Vec<u8>,
|
||||||
@@ -96,9 +93,12 @@ pin_project! {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BitrotWriter {
|
impl<W> BitrotWriter<W>
|
||||||
|
where
|
||||||
|
W: AsyncWrite + Unpin + Send + Sync,
|
||||||
|
{
|
||||||
/// Create a new BitrotWriter.
|
/// Create a new BitrotWriter.
|
||||||
pub fn new(inner: Writer, shard_size: usize, algo: HashAlgorithm) -> Self {
|
pub fn new(inner: W, shard_size: usize, algo: HashAlgorithm) -> Self {
|
||||||
let hash_algo = algo;
|
let hash_algo = algo;
|
||||||
Self {
|
Self {
|
||||||
inner,
|
inner,
|
||||||
@@ -109,7 +109,7 @@ impl BitrotWriter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn into_inner(self) -> Writer {
|
pub fn into_inner(self) -> W {
|
||||||
self.inner
|
self.inner
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,7 +209,7 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
|
||||||
use crate::{BitrotReader, BitrotWriter, Writer};
|
use crate::{BitrotReader, BitrotWriter};
|
||||||
use rustfs_utils::HashAlgorithm;
|
use rustfs_utils::HashAlgorithm;
|
||||||
use std::io::Cursor;
|
use std::io::Cursor;
|
||||||
|
|
||||||
@@ -219,9 +219,9 @@ mod tests {
|
|||||||
let data_size = data.len();
|
let data_size = data.len();
|
||||||
let shard_size = 8;
|
let shard_size = 8;
|
||||||
|
|
||||||
let buf = Vec::new();
|
let buf: Vec<u8> = Vec::new();
|
||||||
let writer = Cursor::new(buf);
|
let writer = Cursor::new(buf);
|
||||||
let mut bitrot_writer = BitrotWriter::new(Writer::from_cursor(writer), shard_size, HashAlgorithm::HighwayHash256);
|
let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256);
|
||||||
|
|
||||||
let mut n = 0;
|
let mut n = 0;
|
||||||
for chunk in data.chunks(shard_size) {
|
for chunk in data.chunks(shard_size) {
|
||||||
@@ -230,8 +230,7 @@ mod tests {
|
|||||||
assert_eq!(n, data.len());
|
assert_eq!(n, data.len());
|
||||||
|
|
||||||
// 读
|
// 读
|
||||||
let reader = Cursor::new(bitrot_writer.into_inner().into_cursor_inner().unwrap());
|
let reader = bitrot_writer.into_inner();
|
||||||
let reader = Box::new(reader);
|
|
||||||
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
|
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
let mut n = 0;
|
let mut n = 0;
|
||||||
@@ -253,18 +252,17 @@ mod tests {
|
|||||||
let data = b"test data for bitrot";
|
let data = b"test data for bitrot";
|
||||||
let data_size = data.len();
|
let data_size = data.len();
|
||||||
let shard_size = 8;
|
let shard_size = 8;
|
||||||
let buf = Vec::new();
|
let buf: Vec<u8> = Vec::new();
|
||||||
let writer = Cursor::new(buf);
|
let writer = Cursor::new(buf);
|
||||||
let mut bitrot_writer = BitrotWriter::new(Writer::from_cursor(writer), shard_size, HashAlgorithm::HighwayHash256);
|
let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256);
|
||||||
for chunk in data.chunks(shard_size) {
|
for chunk in data.chunks(shard_size) {
|
||||||
let _ = bitrot_writer.write(chunk).await.unwrap();
|
let _ = bitrot_writer.write(chunk).await.unwrap();
|
||||||
}
|
}
|
||||||
let mut written = bitrot_writer.into_inner().into_cursor_inner().unwrap();
|
let mut written = bitrot_writer.into_inner().into_inner();
|
||||||
// change the last byte to make hash mismatch
|
// change the last byte to make hash mismatch
|
||||||
let pos = written.len() - 1;
|
let pos = written.len() - 1;
|
||||||
written[pos] ^= 0xFF;
|
written[pos] ^= 0xFF;
|
||||||
let reader = Cursor::new(written);
|
let reader = Cursor::new(written);
|
||||||
let reader = Box::new(reader);
|
|
||||||
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
|
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
|
||||||
|
|
||||||
let count = data_size.div_ceil(shard_size);
|
let count = data_size.div_ceil(shard_size);
|
||||||
@@ -297,9 +295,9 @@ mod tests {
|
|||||||
let data_size = data.len();
|
let data_size = data.len();
|
||||||
let shard_size = 8;
|
let shard_size = 8;
|
||||||
|
|
||||||
let buf = Vec::new();
|
let buf: Vec<u8> = Vec::new();
|
||||||
let writer = Cursor::new(buf);
|
let writer = Cursor::new(buf);
|
||||||
let mut bitrot_writer = BitrotWriter::new(Writer::from_cursor(writer), shard_size, HashAlgorithm::None);
|
let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::None);
|
||||||
|
|
||||||
let mut n = 0;
|
let mut n = 0;
|
||||||
for chunk in data.chunks(shard_size) {
|
for chunk in data.chunks(shard_size) {
|
||||||
@@ -307,8 +305,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
assert_eq!(n, data.len());
|
assert_eq!(n, data.len());
|
||||||
|
|
||||||
let reader = Cursor::new(bitrot_writer.into_inner().into_cursor_inner().unwrap());
|
let reader = bitrot_writer.into_inner();
|
||||||
let reader = Box::new(reader);
|
|
||||||
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::None);
|
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::None);
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
let mut n = 0;
|
let mut n = 0;
|
||||||
|
|||||||
@@ -30,9 +30,6 @@ pub use writer::*;
|
|||||||
mod http_reader;
|
mod http_reader;
|
||||||
pub use http_reader::*;
|
pub use http_reader::*;
|
||||||
|
|
||||||
mod bitrot;
|
|
||||||
pub use bitrot::*;
|
|
||||||
|
|
||||||
mod etag;
|
mod etag;
|
||||||
|
|
||||||
pub trait Reader: tokio::io::AsyncRead + Unpin + Send + Sync + EtagResolvable + HashReaderDetector {}
|
pub trait Reader: tokio::io::AsyncRead + Unpin + Send + Sync + EtagResolvable + HashReaderDetector {}
|
||||||
|
|||||||
+136
-174
@@ -1,308 +1,270 @@
|
|||||||
# Reed-Solomon 纠删码性能基准测试
|
# Reed-Solomon Erasure Coding Performance Benchmark
|
||||||
|
|
||||||
本目录包含了比较不同 Reed-Solomon 实现性能的综合基准测试套件。
|
This directory contains a comprehensive benchmark suite for comparing the performance of different Reed-Solomon implementations.
|
||||||
|
|
||||||
## 📊 测试概述
|
## 📊 Test Overview
|
||||||
|
|
||||||
### 支持的实现模式
|
### Supported Implementation Modes
|
||||||
|
|
||||||
#### 🏛️ 纯 Erasure 模式(默认,推荐)
|
#### 🏛️ Pure Erasure Mode (Default, Recommended)
|
||||||
- **稳定可靠**: 使用成熟的 reed-solomon-erasure 实现
|
- **Stable and Reliable**: Uses mature reed-solomon-erasure implementation
|
||||||
- **广泛兼容**: 支持任意分片大小
|
- **Wide Compatibility**: Supports arbitrary shard sizes
|
||||||
- **内存高效**: 优化的内存使用模式
|
- **Memory Efficient**: Optimized memory usage patterns
|
||||||
- **可预测性**: 性能对分片大小不敏感
|
- **Predictable**: Performance insensitive to shard size
|
||||||
- **使用场景**: 生产环境默认选择,适合大多数应用场景
|
- **Use Case**: Default choice for production environments, suitable for most application scenarios
|
||||||
|
|
||||||
#### 🎯 混合模式(`reed-solomon-simd` feature)
|
#### 🎯 SIMD Mode (`reed-solomon-simd` feature)
|
||||||
- **自动优化**: 根据分片大小智能选择最优实现
|
- **High Performance Optimization**: Uses SIMD instruction sets for high-performance encoding/decoding
|
||||||
- **SIMD + Erasure Fallback**: 大分片使用 SIMD 优化,小分片或 SIMD 失败时自动回退到 Erasure 实现
|
- **Performance Oriented**: Focuses on maximizing processing performance
|
||||||
- **兼容性**: 支持所有分片大小和配置
|
- **Target Scenarios**: High-performance scenarios for large data processing
|
||||||
- **性能**: 在各种场景下都能提供最佳性能
|
- **Use Case**: Scenarios requiring maximum performance, suitable for handling large amounts of data
|
||||||
- **使用场景**: 需要最大化性能的场景,适合处理大量数据
|
|
||||||
|
|
||||||
**回退机制**:
|
### Test Dimensions
|
||||||
- ✅ 分片 ≥ 512 字节:优先使用 SIMD 优化
|
|
||||||
- 🔄 分片 < 512 字节或 SIMD 失败:自动回退到 Erasure 实现
|
|
||||||
- 📊 无缝切换,透明给用户
|
|
||||||
|
|
||||||
### 测试维度
|
- **Encoding Performance** - Speed of encoding data into erasure code shards
|
||||||
|
- **Decoding Performance** - Speed of recovering original data from erasure code shards
|
||||||
|
- **Shard Size Sensitivity** - Impact of different shard sizes on performance
|
||||||
|
- **Erasure Code Configuration** - Performance impact of different data/parity shard ratios
|
||||||
|
- **SIMD Mode Performance** - Performance characteristics of SIMD optimization
|
||||||
|
- **Concurrency Performance** - Performance in multi-threaded environments
|
||||||
|
- **Memory Efficiency** - Memory usage patterns and efficiency
|
||||||
|
- **Error Recovery Capability** - Recovery performance under different numbers of lost shards
|
||||||
|
|
||||||
- **编码性能** - 数据编码成纠删码分片的速度
|
## 🚀 Quick Start
|
||||||
- **解码性能** - 从纠删码分片恢复原始数据的速度
|
|
||||||
- **分片大小敏感性** - 不同分片大小对性能的影响
|
|
||||||
- **纠删码配置** - 不同数据/奇偶分片比例的性能影响
|
|
||||||
- **混合模式回退** - SIMD 与 Erasure 回退机制的性能
|
|
||||||
- **并发性能** - 多线程环境下的性能表现
|
|
||||||
- **内存效率** - 内存使用模式和效率
|
|
||||||
- **错误恢复能力** - 不同丢失分片数量下的恢复性能
|
|
||||||
|
|
||||||
## 🚀 快速开始
|
### Run Quick Tests
|
||||||
|
|
||||||
### 运行快速测试
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 运行快速性能对比测试(默认混合模式)
|
# Run quick performance comparison tests (default pure Erasure mode)
|
||||||
./run_benchmarks.sh quick
|
./run_benchmarks.sh quick
|
||||||
```
|
```
|
||||||
|
|
||||||
### 运行完整对比测试
|
### Run Complete Comparison Tests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 运行详细的实现对比测试
|
# Run detailed implementation comparison tests
|
||||||
./run_benchmarks.sh comparison
|
./run_benchmarks.sh comparison
|
||||||
```
|
```
|
||||||
|
|
||||||
### 运行特定模式的测试
|
### Run Specific Mode Tests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 测试默认纯 erasure 模式(推荐)
|
# Test default pure erasure mode (recommended)
|
||||||
./run_benchmarks.sh erasure
|
./run_benchmarks.sh erasure
|
||||||
|
|
||||||
# 测试混合模式(SIMD + Erasure fallback)
|
# Test SIMD mode
|
||||||
./run_benchmarks.sh hybrid
|
./run_benchmarks.sh simd
|
||||||
```
|
```
|
||||||
|
|
||||||
## 📈 手动运行基准测试
|
## 📈 Manual Benchmark Execution
|
||||||
|
|
||||||
### 基本使用
|
### Basic Usage
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 运行所有基准测试(默认纯 erasure 模式)
|
# Run all benchmarks (default pure erasure mode)
|
||||||
cargo bench
|
cargo bench
|
||||||
|
|
||||||
# 运行特定的基准测试文件
|
# Run specific benchmark files
|
||||||
cargo bench --bench erasure_benchmark
|
cargo bench --bench erasure_benchmark
|
||||||
cargo bench --bench comparison_benchmark
|
cargo bench --bench comparison_benchmark
|
||||||
```
|
```
|
||||||
|
|
||||||
### 对比不同实现模式
|
### Compare Different Implementation Modes
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 测试默认纯 erasure 模式
|
# Test default pure erasure mode
|
||||||
cargo bench --bench comparison_benchmark
|
cargo bench --bench comparison_benchmark
|
||||||
|
|
||||||
# 测试混合模式(SIMD + Erasure fallback)
|
# Test SIMD mode
|
||||||
cargo bench --bench comparison_benchmark \
|
cargo bench --bench comparison_benchmark \
|
||||||
--features reed-solomon-simd
|
--features reed-solomon-simd
|
||||||
|
|
||||||
# 保存基线进行对比
|
# Save baseline for comparison
|
||||||
cargo bench --bench comparison_benchmark \
|
cargo bench --bench comparison_benchmark \
|
||||||
-- --save-baseline erasure_baseline
|
-- --save-baseline erasure_baseline
|
||||||
|
|
||||||
# 与基线比较混合模式性能
|
# Compare SIMD mode performance with baseline
|
||||||
cargo bench --bench comparison_benchmark \
|
cargo bench --bench comparison_benchmark \
|
||||||
--features reed-solomon-simd \
|
--features reed-solomon-simd \
|
||||||
-- --baseline erasure_baseline
|
-- --baseline erasure_baseline
|
||||||
```
|
```
|
||||||
|
|
||||||
### 过滤特定测试
|
### Filter Specific Tests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 只运行编码测试
|
# Run only encoding tests
|
||||||
cargo bench encode
|
cargo bench encode
|
||||||
|
|
||||||
# 只运行解码测试
|
# Run only decoding tests
|
||||||
cargo bench decode
|
cargo bench decode
|
||||||
|
|
||||||
# 只运行特定数据大小的测试
|
# Run tests for specific data sizes
|
||||||
cargo bench 1MB
|
cargo bench 1MB
|
||||||
|
|
||||||
# 只运行特定配置的测试
|
# Run tests for specific configurations
|
||||||
cargo bench "4+2"
|
cargo bench "4+2"
|
||||||
```
|
```
|
||||||
|
|
||||||
## 📊 查看结果
|
## 📊 View Results
|
||||||
|
|
||||||
### HTML 报告
|
### HTML Reports
|
||||||
|
|
||||||
基准测试结果会自动生成 HTML 报告:
|
Benchmark results automatically generate HTML reports:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 启动本地服务器查看报告
|
# Start local server to view reports
|
||||||
cd target/criterion
|
cd target/criterion
|
||||||
python3 -m http.server 8080
|
python3 -m http.server 8080
|
||||||
|
|
||||||
# 在浏览器中访问
|
# Access in browser
|
||||||
open http://localhost:8080/report/index.html
|
open http://localhost:8080/report/index.html
|
||||||
```
|
```
|
||||||
|
|
||||||
### 命令行输出
|
### Command Line Output
|
||||||
|
|
||||||
基准测试会在终端显示:
|
Benchmarks display in terminal:
|
||||||
- 每秒操作数 (ops/sec)
|
- Operations per second (ops/sec)
|
||||||
- 吞吐量 (MB/s)
|
- Throughput (MB/s)
|
||||||
- 延迟统计 (平均值、标准差、百分位数)
|
- Latency statistics (mean, standard deviation, percentiles)
|
||||||
- 性能变化趋势
|
- Performance trend changes
|
||||||
- 回退机制触发情况
|
|
||||||
|
|
||||||
## 🔧 测试配置
|
## 🔧 Test Configuration
|
||||||
|
|
||||||
### 数据大小
|
### Data Sizes
|
||||||
|
|
||||||
- **小数据**: 1KB, 8KB - 测试小文件场景和回退机制
|
- **Small Data**: 1KB, 8KB - Test small file scenarios
|
||||||
- **中等数据**: 64KB, 256KB - 测试常见文件大小
|
- **Medium Data**: 64KB, 256KB - Test common file sizes
|
||||||
- **大数据**: 1MB, 4MB - 测试大文件处理和 SIMD 优化
|
- **Large Data**: 1MB, 4MB - Test large file processing and SIMD optimization
|
||||||
- **超大数据**: 16MB+ - 测试高吞吐量场景
|
- **Very Large Data**: 16MB+ - Test high throughput scenarios
|
||||||
|
|
||||||
### 纠删码配置
|
### Erasure Code Configurations
|
||||||
|
|
||||||
- **(4,2)** - 常用配置,33% 冗余
|
- **(4,2)** - Common configuration, 33% redundancy
|
||||||
- **(6,3)** - 50% 冗余,平衡性能和可靠性
|
- **(6,3)** - 50% redundancy, balanced performance and reliability
|
||||||
- **(8,4)** - 50% 冗余,更多并行度
|
- **(8,4)** - 50% redundancy, more parallelism
|
||||||
- **(10,5)**, **(12,6)** - 高并行度配置
|
- **(10,5)**, **(12,6)** - High parallelism configurations
|
||||||
|
|
||||||
### 分片大小
|
### Shard Sizes
|
||||||
|
|
||||||
测试从 32 字节到 8KB 的不同分片大小,特别关注:
|
Test different shard sizes from 32 bytes to 8KB, with special focus on:
|
||||||
- **回退临界点**: 512 字节 - 混合模式的 SIMD/Erasure 切换点
|
- **Memory Alignment**: 64, 128, 256 bytes - Impact of memory alignment on performance
|
||||||
- **内存对齐**: 64, 128, 256 字节 - 内存对齐对性能的影响
|
- **Cache Friendly**: 1KB, 2KB, 4KB - CPU cache-friendly sizes
|
||||||
- **Cache 友好**: 1KB, 2KB, 4KB - CPU 缓存友好的大小
|
|
||||||
|
|
||||||
## 📝 解读测试结果
|
## 📝 Interpreting Test Results
|
||||||
|
|
||||||
### 性能指标
|
### Performance Metrics
|
||||||
|
|
||||||
1. **吞吐量 (Throughput)**
|
1. **Throughput**
|
||||||
- 单位: MB/s 或 GB/s
|
- Unit: MB/s or GB/s
|
||||||
- 衡量数据处理速度
|
- Measures data processing speed
|
||||||
- 越高越好
|
- Higher is better
|
||||||
|
|
||||||
2. **延迟 (Latency)**
|
2. **Latency**
|
||||||
- 单位: 微秒 (μs) 或毫秒 (ms)
|
- Unit: microseconds (μs) or milliseconds (ms)
|
||||||
- 衡量单次操作时间
|
- Measures single operation time
|
||||||
- 越低越好
|
- Lower is better
|
||||||
|
|
||||||
3. **CPU 效率**
|
3. **CPU Efficiency**
|
||||||
- 每 CPU 周期处理的字节数
|
- Bytes processed per CPU cycle
|
||||||
- 反映算法效率
|
- Reflects algorithm efficiency
|
||||||
|
|
||||||
4. **回退频率**
|
### Expected Results
|
||||||
- 混合模式下 SIMD 到 Erasure 的回退次数
|
|
||||||
- 反映智能选择的效果
|
|
||||||
|
|
||||||
### 预期结果
|
**Pure Erasure Mode (Default)**:
|
||||||
|
- Stable performance, insensitive to shard size
|
||||||
|
- Best compatibility, supports all configurations
|
||||||
|
- Stable and predictable memory usage
|
||||||
|
|
||||||
**纯 Erasure 模式(默认)**:
|
**SIMD Mode (`reed-solomon-simd` feature)**:
|
||||||
- 性能稳定,对分片大小不敏感
|
- High-performance SIMD optimized implementation
|
||||||
- 兼容性最佳,支持所有配置
|
- Suitable for large data processing scenarios
|
||||||
- 内存使用稳定可预测
|
- Focuses on maximizing performance
|
||||||
|
|
||||||
**混合模式(`reed-solomon-simd` feature)**:
|
**Shard Size Sensitivity**:
|
||||||
- 大分片 (≥512B):接近纯 SIMD 性能
|
- SIMD mode may be more sensitive to shard sizes
|
||||||
- 小分片 (<512B):自动回退到 Erasure,保证兼容性
|
- Pure Erasure mode relatively insensitive to shard size
|
||||||
- 整体:在各种场景下都有良好表现
|
|
||||||
|
|
||||||
**分片大小敏感性**:
|
**Memory Usage**:
|
||||||
- 混合模式在 512B 附近可能有性能切换
|
- SIMD mode may have specific memory alignment requirements
|
||||||
- 纯 Erasure 模式对分片大小相对不敏感
|
- Pure Erasure mode has more stable memory usage
|
||||||
|
|
||||||
**内存使用**:
|
## 🛠️ Custom Testing
|
||||||
- 混合模式根据场景优化内存使用
|
|
||||||
- 纯 Erasure 模式内存使用更稳定
|
|
||||||
|
|
||||||
## 🛠️ 自定义测试
|
### Adding New Test Scenarios
|
||||||
|
|
||||||
### 添加新的测试场景
|
Edit `benches/erasure_benchmark.rs` or `benches/comparison_benchmark.rs`:
|
||||||
|
|
||||||
编辑 `benches/erasure_benchmark.rs` 或 `benches/comparison_benchmark.rs`:
|
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
// 添加新的测试配置
|
// Add new test configuration
|
||||||
let configs = vec![
|
let configs = vec![
|
||||||
// 你的自定义配置
|
// Your custom configuration
|
||||||
BenchConfig::new(10, 4, 2048 * 1024, 2048 * 1024), // 10+4, 2MB
|
BenchConfig::new(10, 4, 2048 * 1024, 2048 * 1024), // 10+4, 2MB
|
||||||
];
|
];
|
||||||
```
|
```
|
||||||
|
|
||||||
### 调整测试参数
|
### Adjust Test Parameters
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
// 修改采样和测试时间
|
// Modify sampling and test time
|
||||||
group.sample_size(20); // 样本数量
|
group.sample_size(20); // Sample count
|
||||||
group.measurement_time(Duration::from_secs(10)); // 测试时间
|
group.measurement_time(Duration::from_secs(10)); // Test duration
|
||||||
```
|
```
|
||||||
|
|
||||||
### 测试回退机制
|
## 🐛 Troubleshooting
|
||||||
|
|
||||||
```rust
|
### Common Issues
|
||||||
// 测试混合模式的回退行为
|
|
||||||
#[cfg(not(feature = "reed-solomon-erasure"))]
|
|
||||||
{
|
|
||||||
// 测试小分片是否正确回退
|
|
||||||
let small_data = vec![0u8; 256]; // 小于 512B,应该使用 Erasure
|
|
||||||
let erasure = Erasure::new(4, 2, 256);
|
|
||||||
let result = erasure.encode_data(&small_data);
|
|
||||||
assert!(result.is_ok()); // 应该成功回退
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🐛 故障排除
|
1. **Compilation Errors**: Ensure correct dependencies are installed
|
||||||
|
|
||||||
### 常见问题
|
|
||||||
|
|
||||||
1. **编译错误**: 确保安装了正确的依赖
|
|
||||||
```bash
|
```bash
|
||||||
cargo update
|
cargo update
|
||||||
cargo build --all-features
|
cargo build --all-features
|
||||||
```
|
```
|
||||||
|
|
||||||
2. **性能异常**: 检查是否在正确的模式下运行
|
2. **Performance Anomalies**: Check if running in correct mode
|
||||||
```bash
|
```bash
|
||||||
# 检查当前配置
|
# Check current configuration
|
||||||
cargo bench --bench comparison_benchmark -- --help
|
cargo bench --bench comparison_benchmark -- --help
|
||||||
```
|
```
|
||||||
|
|
||||||
3. **回退过于频繁**: 调整 SIMD 临界点
|
3. **Tests Taking Too Long**: Adjust test parameters
|
||||||
```rust
|
|
||||||
// 在代码中可以调整这个值
|
|
||||||
const SIMD_MIN_SHARD_SIZE: usize = 512;
|
|
||||||
```
|
|
||||||
|
|
||||||
4. **测试时间过长**: 调整测试参数
|
|
||||||
```bash
|
```bash
|
||||||
# 使用更短的测试时间
|
# Use shorter test duration
|
||||||
cargo bench -- --quick
|
cargo bench -- --quick
|
||||||
```
|
```
|
||||||
|
|
||||||
### 性能分析
|
### Performance Analysis
|
||||||
|
|
||||||
使用 `perf` 等工具进行更详细的性能分析:
|
Use tools like `perf` for detailed performance analysis:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 分析 CPU 使用情况
|
# Analyze CPU usage
|
||||||
cargo bench --bench comparison_benchmark &
|
cargo bench --bench comparison_benchmark &
|
||||||
perf record -p $(pgrep -f comparison_benchmark)
|
perf record -p $(pgrep -f comparison_benchmark)
|
||||||
perf report
|
perf report
|
||||||
```
|
```
|
||||||
|
|
||||||
### 调试回退机制
|
## 🤝 Contributing
|
||||||
|
|
||||||
```bash
|
Welcome to submit new benchmark scenarios or optimization suggestions:
|
||||||
# 启用详细日志查看回退情况
|
|
||||||
RUST_LOG=warn cargo bench --bench comparison_benchmark
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🤝 贡献
|
1. Fork the project
|
||||||
|
2. Create feature branch: `git checkout -b feature/new-benchmark`
|
||||||
|
3. Add test cases
|
||||||
|
4. Commit changes: `git commit -m 'Add new benchmark for XYZ'`
|
||||||
|
5. Push to branch: `git push origin feature/new-benchmark`
|
||||||
|
6. Create Pull Request
|
||||||
|
|
||||||
欢迎提交新的基准测试场景或优化建议:
|
## 📚 References
|
||||||
|
|
||||||
1. Fork 项目
|
|
||||||
2. 创建特性分支: `git checkout -b feature/new-benchmark`
|
|
||||||
3. 添加测试用例
|
|
||||||
4. 提交更改: `git commit -m 'Add new benchmark for XYZ'`
|
|
||||||
5. 推送到分支: `git push origin feature/new-benchmark`
|
|
||||||
6. 创建 Pull Request
|
|
||||||
|
|
||||||
## 📚 参考资料
|
|
||||||
|
|
||||||
- [reed-solomon-erasure crate](https://crates.io/crates/reed-solomon-erasure)
|
- [reed-solomon-erasure crate](https://crates.io/crates/reed-solomon-erasure)
|
||||||
- [reed-solomon-simd crate](https://crates.io/crates/reed-solomon-simd)
|
- [reed-solomon-simd crate](https://crates.io/crates/reed-solomon-simd)
|
||||||
- [Criterion.rs 基准测试框架](https://bheisler.github.io/criterion.rs/book/)
|
- [Criterion.rs benchmark framework](https://bheisler.github.io/criterion.rs/book/)
|
||||||
- [Reed-Solomon 纠删码原理](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction)
|
- [Reed-Solomon error correction principles](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
💡 **提示**:
|
💡 **Tips**:
|
||||||
- 推荐使用默认的混合模式,它能在各种场景下自动选择最优实现
|
- Recommend using the default pure Erasure mode, which provides stable performance across various scenarios
|
||||||
- 基准测试结果可能因硬件、操作系统和编译器版本而异
|
- Consider SIMD mode for high-performance requirements
|
||||||
- 建议在目标部署环境中运行测试以获得最准确的性能数据
|
- Benchmark results may vary based on hardware, operating system, and compiler versions
|
||||||
|
- Suggest running tests in target deployment environment for most accurate performance data
|
||||||
@@ -0,0 +1,270 @@
|
|||||||
|
# Reed-Solomon 纠删码性能基准测试
|
||||||
|
|
||||||
|
本目录包含了比较不同 Reed-Solomon 实现性能的综合基准测试套件。
|
||||||
|
|
||||||
|
## 📊 测试概述
|
||||||
|
|
||||||
|
### 支持的实现模式
|
||||||
|
|
||||||
|
#### 🏛️ 纯 Erasure 模式(默认,推荐)
|
||||||
|
- **稳定可靠**: 使用成熟的 reed-solomon-erasure 实现
|
||||||
|
- **广泛兼容**: 支持任意分片大小
|
||||||
|
- **内存高效**: 优化的内存使用模式
|
||||||
|
- **可预测性**: 性能对分片大小不敏感
|
||||||
|
- **使用场景**: 生产环境默认选择,适合大多数应用场景
|
||||||
|
|
||||||
|
#### 🎯 SIMD模式(`reed-solomon-simd` feature)
|
||||||
|
- **高性能优化**: 使用SIMD指令集进行高性能编码解码
|
||||||
|
- **性能导向**: 专注于最大化处理性能
|
||||||
|
- **适用场景**: 大数据量处理的高性能场景
|
||||||
|
- **使用场景**: 需要最大化性能的场景,适合处理大量数据
|
||||||
|
|
||||||
|
### 测试维度
|
||||||
|
|
||||||
|
- **编码性能** - 数据编码成纠删码分片的速度
|
||||||
|
- **解码性能** - 从纠删码分片恢复原始数据的速度
|
||||||
|
- **分片大小敏感性** - 不同分片大小对性能的影响
|
||||||
|
- **纠删码配置** - 不同数据/奇偶分片比例的性能影响
|
||||||
|
- **SIMD模式性能** - SIMD优化的性能表现
|
||||||
|
- **并发性能** - 多线程环境下的性能表现
|
||||||
|
- **内存效率** - 内存使用模式和效率
|
||||||
|
- **错误恢复能力** - 不同丢失分片数量下的恢复性能
|
||||||
|
|
||||||
|
## 🚀 快速开始
|
||||||
|
|
||||||
|
### 运行快速测试
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 运行快速性能对比测试(默认纯Erasure模式)
|
||||||
|
./run_benchmarks.sh quick
|
||||||
|
```
|
||||||
|
|
||||||
|
### 运行完整对比测试
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 运行详细的实现对比测试
|
||||||
|
./run_benchmarks.sh comparison
|
||||||
|
```
|
||||||
|
|
||||||
|
### 运行特定模式的测试
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 测试默认纯 erasure 模式(推荐)
|
||||||
|
./run_benchmarks.sh erasure
|
||||||
|
|
||||||
|
# 测试SIMD模式
|
||||||
|
./run_benchmarks.sh simd
|
||||||
|
```
|
||||||
|
|
||||||
|
## 📈 手动运行基准测试
|
||||||
|
|
||||||
|
### 基本使用
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 运行所有基准测试(默认纯 erasure 模式)
|
||||||
|
cargo bench
|
||||||
|
|
||||||
|
# 运行特定的基准测试文件
|
||||||
|
cargo bench --bench erasure_benchmark
|
||||||
|
cargo bench --bench comparison_benchmark
|
||||||
|
```
|
||||||
|
|
||||||
|
### 对比不同实现模式
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 测试默认纯 erasure 模式
|
||||||
|
cargo bench --bench comparison_benchmark
|
||||||
|
|
||||||
|
# 测试SIMD模式
|
||||||
|
cargo bench --bench comparison_benchmark \
|
||||||
|
--features reed-solomon-simd
|
||||||
|
|
||||||
|
# 保存基线进行对比
|
||||||
|
cargo bench --bench comparison_benchmark \
|
||||||
|
-- --save-baseline erasure_baseline
|
||||||
|
|
||||||
|
# 与基线比较SIMD模式性能
|
||||||
|
cargo bench --bench comparison_benchmark \
|
||||||
|
--features reed-solomon-simd \
|
||||||
|
-- --baseline erasure_baseline
|
||||||
|
```
|
||||||
|
|
||||||
|
### 过滤特定测试
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 只运行编码测试
|
||||||
|
cargo bench encode
|
||||||
|
|
||||||
|
# 只运行解码测试
|
||||||
|
cargo bench decode
|
||||||
|
|
||||||
|
# 只运行特定数据大小的测试
|
||||||
|
cargo bench 1MB
|
||||||
|
|
||||||
|
# 只运行特定配置的测试
|
||||||
|
cargo bench "4+2"
|
||||||
|
```
|
||||||
|
|
||||||
|
## 📊 查看结果
|
||||||
|
|
||||||
|
### HTML 报告
|
||||||
|
|
||||||
|
基准测试结果会自动生成 HTML 报告:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 启动本地服务器查看报告
|
||||||
|
cd target/criterion
|
||||||
|
python3 -m http.server 8080
|
||||||
|
|
||||||
|
# 在浏览器中访问
|
||||||
|
open http://localhost:8080/report/index.html
|
||||||
|
```
|
||||||
|
|
||||||
|
### 命令行输出
|
||||||
|
|
||||||
|
基准测试会在终端显示:
|
||||||
|
- 每秒操作数 (ops/sec)
|
||||||
|
- 吞吐量 (MB/s)
|
||||||
|
- 延迟统计 (平均值、标准差、百分位数)
|
||||||
|
- 性能变化趋势
|
||||||
|
|
||||||
|
## 🔧 测试配置
|
||||||
|
|
||||||
|
### 数据大小
|
||||||
|
|
||||||
|
- **小数据**: 1KB, 8KB - 测试小文件场景
|
||||||
|
- **中等数据**: 64KB, 256KB - 测试常见文件大小
|
||||||
|
- **大数据**: 1MB, 4MB - 测试大文件处理和 SIMD 优化
|
||||||
|
- **超大数据**: 16MB+ - 测试高吞吐量场景
|
||||||
|
|
||||||
|
### 纠删码配置
|
||||||
|
|
||||||
|
- **(4,2)** - 常用配置,33% 冗余
|
||||||
|
- **(6,3)** - 50% 冗余,平衡性能和可靠性
|
||||||
|
- **(8,4)** - 50% 冗余,更多并行度
|
||||||
|
- **(10,5)**, **(12,6)** - 高并行度配置
|
||||||
|
|
||||||
|
### 分片大小
|
||||||
|
|
||||||
|
测试从 32 字节到 8KB 的不同分片大小,特别关注:
|
||||||
|
- **内存对齐**: 64, 128, 256 字节 - 内存对齐对性能的影响
|
||||||
|
- **Cache 友好**: 1KB, 2KB, 4KB - CPU 缓存友好的大小
|
||||||
|
|
||||||
|
## 📝 解读测试结果
|
||||||
|
|
||||||
|
### 性能指标
|
||||||
|
|
||||||
|
1. **吞吐量 (Throughput)**
|
||||||
|
- 单位: MB/s 或 GB/s
|
||||||
|
- 衡量数据处理速度
|
||||||
|
- 越高越好
|
||||||
|
|
||||||
|
2. **延迟 (Latency)**
|
||||||
|
- 单位: 微秒 (μs) 或毫秒 (ms)
|
||||||
|
- 衡量单次操作时间
|
||||||
|
- 越低越好
|
||||||
|
|
||||||
|
3. **CPU 效率**
|
||||||
|
- 每 CPU 周期处理的字节数
|
||||||
|
- 反映算法效率
|
||||||
|
|
||||||
|
### 预期结果
|
||||||
|
|
||||||
|
**纯 Erasure 模式(默认)**:
|
||||||
|
- 性能稳定,对分片大小不敏感
|
||||||
|
- 兼容性最佳,支持所有配置
|
||||||
|
- 内存使用稳定可预测
|
||||||
|
|
||||||
|
**SIMD模式(`reed-solomon-simd` feature)**:
|
||||||
|
- 高性能SIMD优化实现
|
||||||
|
- 适合大数据量处理场景
|
||||||
|
- 专注于最大化性能
|
||||||
|
|
||||||
|
**分片大小敏感性**:
|
||||||
|
- SIMD模式对分片大小可能更敏感
|
||||||
|
- 纯 Erasure 模式对分片大小相对不敏感
|
||||||
|
|
||||||
|
**内存使用**:
|
||||||
|
- SIMD模式可能有特定的内存对齐要求
|
||||||
|
- 纯 Erasure 模式内存使用更稳定
|
||||||
|
|
||||||
|
## 🛠️ 自定义测试
|
||||||
|
|
||||||
|
### 添加新的测试场景
|
||||||
|
|
||||||
|
编辑 `benches/erasure_benchmark.rs` 或 `benches/comparison_benchmark.rs`:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// 添加新的测试配置
|
||||||
|
let configs = vec![
|
||||||
|
// 你的自定义配置
|
||||||
|
BenchConfig::new(10, 4, 2048 * 1024, 2048 * 1024), // 10+4, 2MB
|
||||||
|
];
|
||||||
|
```
|
||||||
|
|
||||||
|
### 调整测试参数
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// 修改采样和测试时间
|
||||||
|
group.sample_size(20); // 样本数量
|
||||||
|
group.measurement_time(Duration::from_secs(10)); // 测试时间
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🐛 故障排除
|
||||||
|
|
||||||
|
### 常见问题
|
||||||
|
|
||||||
|
1. **编译错误**: 确保安装了正确的依赖
|
||||||
|
```bash
|
||||||
|
cargo update
|
||||||
|
cargo build --all-features
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **性能异常**: 检查是否在正确的模式下运行
|
||||||
|
```bash
|
||||||
|
# 检查当前配置
|
||||||
|
cargo bench --bench comparison_benchmark -- --help
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **测试时间过长**: 调整测试参数
|
||||||
|
```bash
|
||||||
|
# 使用更短的测试时间
|
||||||
|
cargo bench -- --quick
|
||||||
|
```
|
||||||
|
|
||||||
|
### 性能分析
|
||||||
|
|
||||||
|
使用 `perf` 等工具进行更详细的性能分析:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 分析 CPU 使用情况
|
||||||
|
cargo bench --bench comparison_benchmark &
|
||||||
|
perf record -p $(pgrep -f comparison_benchmark)
|
||||||
|
perf report
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🤝 贡献
|
||||||
|
|
||||||
|
欢迎提交新的基准测试场景或优化建议:
|
||||||
|
|
||||||
|
1. Fork 项目
|
||||||
|
2. 创建特性分支: `git checkout -b feature/new-benchmark`
|
||||||
|
3. 添加测试用例
|
||||||
|
4. 提交更改: `git commit -m 'Add new benchmark for XYZ'`
|
||||||
|
5. 推送到分支: `git push origin feature/new-benchmark`
|
||||||
|
6. 创建 Pull Request
|
||||||
|
|
||||||
|
## 📚 参考资料
|
||||||
|
|
||||||
|
- [reed-solomon-erasure crate](https://crates.io/crates/reed-solomon-erasure)
|
||||||
|
- [reed-solomon-simd crate](https://crates.io/crates/reed-solomon-simd)
|
||||||
|
- [Criterion.rs 基准测试框架](https://bheisler.github.io/criterion.rs/book/)
|
||||||
|
- [Reed-Solomon 纠删码原理](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
💡 **提示**:
|
||||||
|
- 推荐使用默认的纯Erasure模式,它在各种场景下都有稳定的表现
|
||||||
|
- 对于高性能需求可以考虑SIMD模式
|
||||||
|
- 基准测试结果可能因硬件、操作系统和编译器版本而异
|
||||||
|
- 建议在目标部署环境中运行测试以获得最准确的性能数据
|
||||||
+1
-1
@@ -11,7 +11,7 @@ rust-version.workspace = true
|
|||||||
workspace = true
|
workspace = true
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = ["reed-solomon-erasure"]
|
default = ["reed-solomon-simd"]
|
||||||
reed-solomon-simd = []
|
reed-solomon-simd = []
|
||||||
reed-solomon-erasure = []
|
reed-solomon-erasure = []
|
||||||
|
|
||||||
|
|||||||
@@ -1,97 +1,69 @@
|
|||||||
# Reed-Solomon 实现对比分析
|
# Reed-Solomon Implementation Comparison Analysis
|
||||||
|
|
||||||
## 🔍 问题分析
|
## 🔍 Issue Analysis
|
||||||
|
|
||||||
随着新的混合模式设计,我们已经解决了传统纯 SIMD 模式的兼容性问题。现在系统能够智能地在不同场景下选择最优实现。
|
With the optimized SIMD mode design, we provide high-performance Reed-Solomon implementation. The system can now deliver optimal performance across different scenarios.
|
||||||
|
|
||||||
## 📊 实现模式对比
|
## 📊 Implementation Mode Comparison
|
||||||
|
|
||||||
### 🏛️ 纯 Erasure 模式(默认,推荐)
|
### 🏛️ Pure Erasure Mode (Default, Recommended)
|
||||||
|
|
||||||
**默认配置**: 不指定任何 feature,使用稳定的 reed-solomon-erasure 实现
|
**Default Configuration**: No features specified, uses stable reed-solomon-erasure implementation
|
||||||
|
|
||||||
**特点**:
|
**Characteristics**:
|
||||||
- ✅ **广泛兼容**: 支持任意分片大小,从字节级到 GB 级
|
- ✅ **Wide Compatibility**: Supports any shard size from byte-level to GB-level
|
||||||
- 📈 **稳定性能**: 性能对分片大小不敏感,可预测
|
- 📈 **Stable Performance**: Performance insensitive to shard size, predictable
|
||||||
- 🔧 **生产就绪**: 成熟稳定的实现,已在生产环境广泛使用
|
- 🔧 **Production Ready**: Mature and stable implementation, widely used in production
|
||||||
- 💾 **内存高效**: 优化的内存使用模式
|
- 💾 **Memory Efficient**: Optimized memory usage patterns
|
||||||
- 🎯 **一致性**: 在所有场景下行为完全一致
|
- 🎯 **Consistency**: Completely consistent behavior across all scenarios
|
||||||
|
|
||||||
**使用场景**:
|
**Use Cases**:
|
||||||
- 大多数生产环境的默认选择
|
- Default choice for most production environments
|
||||||
- 需要完全一致和可预测的性能行为
|
- Systems requiring completely consistent and predictable performance behavior
|
||||||
- 对性能变化敏感的系统
|
- Performance-change-sensitive systems
|
||||||
- 主要处理小文件或小分片的场景
|
- Scenarios mainly processing small files or small shards
|
||||||
- 需要严格的内存使用控制
|
- Systems requiring strict memory usage control
|
||||||
|
|
||||||
### 🎯 混合模式(`reed-solomon-simd` feature)
|
### 🎯 SIMD Mode (`reed-solomon-simd` feature)
|
||||||
|
|
||||||
**配置**: `--features reed-solomon-simd`
|
**Configuration**: `--features reed-solomon-simd`
|
||||||
|
|
||||||
**特点**:
|
**Characteristics**:
|
||||||
- 🧠 **智能选择**: 根据分片大小自动选择 SIMD 或 Erasure 实现
|
- 🚀 **High-Performance SIMD**: Uses SIMD instruction sets for high-performance encoding/decoding
|
||||||
- 🚀 **最优性能**: 大分片使用 SIMD 优化,小分片使用稳定的 Erasure 实现
|
- 🎯 **Performance Oriented**: Focuses on maximizing processing performance
|
||||||
- 🔄 **自动回退**: SIMD 失败时无缝回退到 Erasure 实现
|
- ⚡ **Large Data Optimization**: Suitable for high-throughput scenarios with large data processing
|
||||||
- ✅ **全兼容**: 支持所有分片大小和配置,无失败风险
|
- 🏎️ **Speed Priority**: Designed for performance-critical applications
|
||||||
- 🎯 **高性能**: 适合需要最大化性能的场景
|
|
||||||
|
|
||||||
**回退逻辑**:
|
**Use Cases**:
|
||||||
```rust
|
- Application scenarios requiring maximum performance
|
||||||
const SIMD_MIN_SHARD_SIZE: usize = 512;
|
- High-throughput systems processing large amounts of data
|
||||||
|
- Scenarios with extremely high performance requirements
|
||||||
|
- CPU-intensive workloads
|
||||||
|
|
||||||
// 智能选择策略
|
## 📏 Shard Size vs Performance Comparison
|
||||||
if shard_len >= SIMD_MIN_SHARD_SIZE {
|
|
||||||
// 尝试使用 SIMD 优化
|
|
||||||
match simd_encode(data) {
|
|
||||||
Ok(result) => return Ok(result),
|
|
||||||
Err(_) => {
|
|
||||||
// SIMD 失败,自动回退到 Erasure
|
|
||||||
warn!("SIMD failed, falling back to Erasure");
|
|
||||||
erasure_encode(data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// 分片太小,直接使用 Erasure
|
|
||||||
erasure_encode(data)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**成功案例**:
|
Performance across different configurations:
|
||||||
```
|
|
||||||
✅ 1KB 数据 + 6+3 配置 → 171字节/分片 → 自动使用 Erasure 实现
|
|
||||||
✅ 64KB 数据 + 4+2 配置 → 16KB/分片 → 自动使用 SIMD 优化
|
|
||||||
✅ 任意配置 → 智能选择最优实现
|
|
||||||
```
|
|
||||||
|
|
||||||
**使用场景**:
|
| Data Size | Config | Shard Size | Pure Erasure Mode (Default) | SIMD Mode Strategy | Performance Comparison |
|
||||||
- 需要最大化性能的应用场景
|
|-----------|--------|------------|----------------------------|-------------------|----------------------|
|
||||||
- 处理大量数据的高吞吐量系统
|
| 1KB | 4+2 | 256 bytes | Erasure implementation | SIMD implementation | SIMD may be faster |
|
||||||
- 对性能要求极高的场景
|
| 1KB | 6+3 | 171 bytes | Erasure implementation | SIMD implementation | SIMD may be faster |
|
||||||
|
| 1KB | 8+4 | 128 bytes | Erasure implementation | SIMD implementation | SIMD may be faster |
|
||||||
|
| 64KB | 4+2 | 16KB | Erasure implementation | SIMD optimization | SIMD mode faster |
|
||||||
|
| 64KB | 6+3 | 10.7KB | Erasure implementation | SIMD optimization | SIMD mode faster |
|
||||||
|
| 1MB | 4+2 | 256KB | Erasure implementation | SIMD optimization | SIMD mode significantly faster |
|
||||||
|
| 16MB | 8+4 | 2MB | Erasure implementation | SIMD optimization | SIMD mode substantially faster |
|
||||||
|
|
||||||
## 📏 分片大小与性能对比
|
## 🎯 Benchmark Results Interpretation
|
||||||
|
|
||||||
不同配置下的性能表现:
|
### Pure Erasure Mode Example (Default) ✅
|
||||||
|
|
||||||
| 数据大小 | 配置 | 分片大小 | 纯 Erasure 模式(默认) | 混合模式策略 | 性能对比 |
|
|
||||||
|---------|------|----------|------------------------|-------------|----------|
|
|
||||||
| 1KB | 4+2 | 256字节 | Erasure 实现 | Erasure 实现 | 相同 |
|
|
||||||
| 1KB | 6+3 | 171字节 | Erasure 实现 | Erasure 实现 | 相同 |
|
|
||||||
| 1KB | 8+4 | 128字节 | Erasure 实现 | Erasure 实现 | 相同 |
|
|
||||||
| 64KB | 4+2 | 16KB | Erasure 实现 | SIMD 优化 | 混合模式更快 |
|
|
||||||
| 64KB | 6+3 | 10.7KB | Erasure 实现 | SIMD 优化 | 混合模式更快 |
|
|
||||||
| 1MB | 4+2 | 256KB | Erasure 实现 | SIMD 优化 | 混合模式显著更快 |
|
|
||||||
| 16MB | 8+4 | 2MB | Erasure 实现 | SIMD 优化 | 混合模式大幅领先 |
|
|
||||||
|
|
||||||
## 🎯 基准测试结果解读
|
|
||||||
|
|
||||||
### 纯 Erasure 模式示例(默认) ✅
|
|
||||||
|
|
||||||
```
|
```
|
||||||
encode_comparison/implementation/1KB_6+3_erasure
|
encode_comparison/implementation/1KB_6+3_erasure
|
||||||
time: [245.67 ns 256.78 ns 267.89 ns]
|
time: [245.67 ns 256.78 ns 267.89 ns]
|
||||||
thrpt: [3.73 GiB/s 3.89 GiB/s 4.07 GiB/s]
|
thrpt: [3.73 GiB/s 3.89 GiB/s 4.07 GiB/s]
|
||||||
|
|
||||||
💡 一致的 Erasure 性能 - 所有配置都使用相同实现
|
💡 Consistent Erasure performance - All configurations use the same implementation
|
||||||
```
|
```
|
||||||
|
|
||||||
```
|
```
|
||||||
@@ -99,262 +71,263 @@ encode_comparison/implementation/64KB_4+2_erasure
|
|||||||
time: [2.3456 μs 2.4567 μs 2.5678 μs]
|
time: [2.3456 μs 2.4567 μs 2.5678 μs]
|
||||||
thrpt: [23.89 GiB/s 24.65 GiB/s 25.43 GiB/s]
|
thrpt: [23.89 GiB/s 24.65 GiB/s 25.43 GiB/s]
|
||||||
|
|
||||||
💡 稳定可靠的性能 - 适合大多数生产场景
|
💡 Stable and reliable performance - Suitable for most production scenarios
|
||||||
```
|
```
|
||||||
|
|
||||||
### 混合模式成功示例 ✅
|
### SIMD Mode Success Examples ✅
|
||||||
|
|
||||||
**大分片 SIMD 优化**:
|
**Large Shard SIMD Optimization**:
|
||||||
```
|
```
|
||||||
encode_comparison/implementation/64KB_4+2_hybrid
|
encode_comparison/implementation/64KB_4+2_simd
|
||||||
time: [1.2345 μs 1.2567 μs 1.2789 μs]
|
time: [1.2345 μs 1.2567 μs 1.2789 μs]
|
||||||
thrpt: [47.89 GiB/s 48.65 GiB/s 49.43 GiB/s]
|
thrpt: [47.89 GiB/s 48.65 GiB/s 49.43 GiB/s]
|
||||||
|
|
||||||
💡 使用 SIMD 优化 - 分片大小: 16KB ≥ 512字节
|
💡 Using SIMD optimization - Shard size: 16KB, high-performance processing
|
||||||
```
|
```
|
||||||
|
|
||||||
**小分片智能回退**:
|
**Small Shard SIMD Processing**:
|
||||||
```
|
```
|
||||||
encode_comparison/implementation/1KB_6+3_hybrid
|
encode_comparison/implementation/1KB_6+3_simd
|
||||||
time: [234.56 ns 245.67 ns 256.78 ns]
|
time: [234.56 ns 245.67 ns 256.78 ns]
|
||||||
thrpt: [3.89 GiB/s 4.07 GiB/s 4.26 GiB/s]
|
thrpt: [3.89 GiB/s 4.07 GiB/s 4.26 GiB/s]
|
||||||
|
|
||||||
💡 智能回退到 Erasure - 分片大小: 171字节 < 512字节
|
💡 SIMD processing small shards - Shard size: 171 bytes
|
||||||
```
|
```
|
||||||
|
|
||||||
**回退机制触发**:
|
## 🛠️ Usage Guide
|
||||||
```
|
|
||||||
⚠️ SIMD encoding failed: InvalidShardSize, using fallback
|
|
||||||
✅ Fallback to Erasure successful - 无缝处理
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🛠️ 使用指南
|
### Selection Strategy
|
||||||
|
|
||||||
### 选择策略
|
#### 1️⃣ Recommended: Pure Erasure Mode (Default)
|
||||||
|
|
||||||
#### 1️⃣ 推荐:纯 Erasure 模式(默认)
|
|
||||||
```bash
|
```bash
|
||||||
# 无需指定 feature,使用默认配置
|
# No features needed, use default configuration
|
||||||
cargo run
|
cargo run
|
||||||
cargo test
|
cargo test
|
||||||
cargo bench
|
cargo bench
|
||||||
```
|
```
|
||||||
|
|
||||||
**适用场景**:
|
**Applicable Scenarios**:
|
||||||
- 📊 **一致性要求**: 需要完全可预测的性能行为
|
- 📊 **Consistency Requirements**: Need completely predictable performance behavior
|
||||||
- 🔬 **生产环境**: 大多数生产场景的最佳选择
|
- 🔬 **Production Environment**: Best choice for most production scenarios
|
||||||
- 💾 **内存敏感**: 对内存使用模式有严格要求
|
- 💾 **Memory Sensitive**: Strict requirements for memory usage patterns
|
||||||
- 🏗️ **稳定可靠**: 成熟稳定的实现
|
- 🏗️ **Stable and Reliable**: Mature and stable implementation
|
||||||
|
|
||||||
#### 2️⃣ 高性能需求:混合模式
|
#### 2️⃣ High Performance Requirements: SIMD Mode
|
||||||
```bash
|
```bash
|
||||||
# 启用混合模式获得最大性能
|
# Enable SIMD mode for maximum performance
|
||||||
cargo run --features reed-solomon-simd
|
cargo run --features reed-solomon-simd
|
||||||
cargo test --features reed-solomon-simd
|
cargo test --features reed-solomon-simd
|
||||||
cargo bench --features reed-solomon-simd
|
cargo bench --features reed-solomon-simd
|
||||||
```
|
```
|
||||||
|
|
||||||
**适用场景**:
|
**Applicable Scenarios**:
|
||||||
- 🎯 **高性能场景**: 处理大量数据需要最大吞吐量
|
- 🎯 **High Performance Scenarios**: Processing large amounts of data requiring maximum throughput
|
||||||
- 🚀 **性能优化**: 希望在大数据时获得最佳性能
|
- 🚀 **Performance Optimization**: Want optimal performance for large data
|
||||||
- 🔄 **智能适应**: 让系统自动选择最优策略
|
- ⚡ **Speed Priority**: Scenarios with extremely high speed requirements
|
||||||
- 🛡️ **容错能力**: 需要最大的兼容性和稳定性
|
- 🏎️ **Compute Intensive**: CPU-intensive workloads
|
||||||
|
|
||||||
### 配置优化建议
|
### Configuration Optimization Recommendations
|
||||||
|
|
||||||
#### 针对数据大小的配置
|
#### Based on Data Size
|
||||||
|
|
||||||
**小文件为主** (< 64KB):
|
**Small Files Primarily** (< 64KB):
|
||||||
```toml
|
```toml
|
||||||
# 推荐使用默认纯 Erasure 模式
|
# Recommended to use default pure Erasure mode
|
||||||
# 无需特殊配置,性能稳定可靠
|
# No special configuration needed, stable and reliable performance
|
||||||
```
|
```
|
||||||
|
|
||||||
**大文件为主** (> 1MB):
|
**Large Files Primarily** (> 1MB):
|
||||||
```toml
|
```toml
|
||||||
# 可考虑启用混合模式获得更高性能
|
# Recommend enabling SIMD mode for higher performance
|
||||||
# features = ["reed-solomon-simd"]
|
# features = ["reed-solomon-simd"]
|
||||||
```
|
```
|
||||||
|
|
||||||
**混合场景**:
|
**Mixed Scenarios**:
|
||||||
```toml
|
```toml
|
||||||
# 默认纯 Erasure 模式适合大多数场景
|
# Default pure Erasure mode suits most scenarios
|
||||||
# 如需最大性能可启用: features = ["reed-solomon-simd"]
|
# For maximum performance, enable: features = ["reed-solomon-simd"]
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 针对纠删码配置的建议
|
#### Recommendations Based on Erasure Coding Configuration
|
||||||
|
|
||||||
| 配置 | 小数据 (< 64KB) | 大数据 (> 1MB) | 推荐模式 |
|
| Config | Small Data (< 64KB) | Large Data (> 1MB) | Recommended Mode |
|
||||||
|------|----------------|----------------|----------|
|
|--------|-------------------|-------------------|------------------|
|
||||||
| 4+2 | 纯 Erasure | 纯 Erasure / 混合模式 | 纯 Erasure(默认) |
|
| 4+2 | Pure Erasure | Pure Erasure / SIMD Mode | Pure Erasure (Default) |
|
||||||
| 6+3 | 纯 Erasure | 纯 Erasure / 混合模式 | 纯 Erasure(默认) |
|
| 6+3 | Pure Erasure | Pure Erasure / SIMD Mode | Pure Erasure (Default) |
|
||||||
| 8+4 | 纯 Erasure | 纯 Erasure / 混合模式 | 纯 Erasure(默认) |
|
| 8+4 | Pure Erasure | Pure Erasure / SIMD Mode | Pure Erasure (Default) |
|
||||||
| 10+5 | 纯 Erasure | 纯 Erasure / 混合模式 | 纯 Erasure(默认) |
|
| 10+5 | Pure Erasure | Pure Erasure / SIMD Mode | Pure Erasure (Default) |
|
||||||
|
|
||||||
### 生产环境部署建议
|
### Production Environment Deployment Recommendations
|
||||||
|
|
||||||
#### 1️⃣ 默认部署策略
|
#### 1️⃣ Default Deployment Strategy
|
||||||
```bash
|
```bash
|
||||||
# 生产环境推荐配置:使用纯 Erasure 模式(默认)
|
# Production environment recommended configuration: Use pure Erasure mode (default)
|
||||||
cargo build --release
|
cargo build --release
|
||||||
```
|
```
|
||||||
|
|
||||||
**优势**:
|
**Advantages**:
|
||||||
- ✅ 最大兼容性:处理任意大小数据
|
- ✅ Maximum compatibility: Handle data of any size
|
||||||
- ✅ 稳定可靠:成熟的实现,行为可预测
|
- ✅ Stable and reliable: Mature implementation, predictable behavior
|
||||||
- ✅ 零配置:无需复杂的性能调优
|
- ✅ Zero configuration: No complex performance tuning needed
|
||||||
- ✅ 内存高效:优化的内存使用模式
|
- ✅ Memory efficient: Optimized memory usage patterns
|
||||||
|
|
||||||
#### 2️⃣ 高性能部署策略
|
#### 2️⃣ High Performance Deployment Strategy
|
||||||
```bash
|
```bash
|
||||||
# 高性能场景:启用混合模式
|
# High performance scenarios: Enable SIMD mode
|
||||||
cargo build --release --features reed-solomon-simd
|
cargo build --release --features reed-solomon-simd
|
||||||
```
|
```
|
||||||
|
|
||||||
**优势**:
|
**Advantages**:
|
||||||
- ✅ 最优性能:自动选择最佳实现
|
- ✅ Optimal performance: SIMD instruction set optimization
|
||||||
- ✅ 智能回退:SIMD 失败自动回退到 Erasure
|
- ✅ High throughput: Suitable for large data processing
|
||||||
- ✅ 大数据优化:大分片自动使用 SIMD 优化
|
- ✅ Performance oriented: Focuses on maximizing processing speed
|
||||||
- ✅ 兼容保证:小分片使用稳定的 Erasure 实现
|
- ✅ Modern hardware: Fully utilizes modern CPU features
|
||||||
|
|
||||||
#### 2️⃣ 监控和调优
|
#### 2️⃣ Monitoring and Tuning
|
||||||
```rust
|
```rust
|
||||||
// 启用警告日志查看回退情况
|
// Choose appropriate implementation based on specific scenarios
|
||||||
RUST_LOG=warn ./your_application
|
match data_size {
|
||||||
|
size if size > 1024 * 1024 => {
|
||||||
// 典型日志输出
|
// Large data: Consider using SIMD mode
|
||||||
warn!("SIMD encoding failed: InvalidShardSize, using fallback");
|
println!("Large data detected, SIMD mode recommended");
|
||||||
info!("Smart fallback to Erasure successful");
|
}
|
||||||
```
|
_ => {
|
||||||
|
// General case: Use default Erasure mode
|
||||||
#### 3️⃣ 性能监控指标
|
println!("Using default Erasure mode");
|
||||||
- **回退频率**: 监控 SIMD 到 Erasure 的回退次数
|
}
|
||||||
- **性能分布**: 观察不同数据大小的性能表现
|
|
||||||
- **内存使用**: 监控内存分配模式
|
|
||||||
- **延迟分布**: 分析编码/解码延迟的统计分布
|
|
||||||
|
|
||||||
## 🔧 故障排除
|
|
||||||
|
|
||||||
### 性能问题诊断
|
|
||||||
|
|
||||||
#### 问题1: 性能不稳定
|
|
||||||
**现象**: 相同操作的性能差异很大
|
|
||||||
**原因**: 可能在 SIMD/Erasure 切换边界附近
|
|
||||||
**解决**:
|
|
||||||
```rust
|
|
||||||
// 检查分片大小
|
|
||||||
let shard_size = data.len().div_ceil(data_shards);
|
|
||||||
println!("Shard size: {} bytes", shard_size);
|
|
||||||
if shard_size >= 512 {
|
|
||||||
println!("Expected to use SIMD optimization");
|
|
||||||
} else {
|
|
||||||
println!("Expected to use Erasure fallback");
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 问题2: 意外的回退行为
|
#### 3️⃣ Performance Monitoring Metrics
|
||||||
**现象**: 大分片仍然使用 Erasure 实现
|
- **Throughput Monitoring**: Monitor encoding/decoding data processing rates
|
||||||
**原因**: SIMD 初始化失败或系统限制
|
- **Latency Analysis**: Analyze processing latency for different data sizes
|
||||||
**解决**:
|
- **CPU Utilization**: Observe CPU utilization efficiency of SIMD instructions
|
||||||
```bash
|
- **Memory Usage**: Monitor memory allocation patterns of different implementations
|
||||||
# 启用详细日志查看回退原因
|
|
||||||
RUST_LOG=debug ./your_application
|
## 🔧 Troubleshooting
|
||||||
|
|
||||||
|
### Performance Issue Diagnosis
|
||||||
|
|
||||||
|
#### Issue 1: Performance Not Meeting Expectations
|
||||||
|
**Symptom**: SIMD mode performance improvement not significant
|
||||||
|
**Cause**: Data size may not be suitable for SIMD optimization
|
||||||
|
**Solution**:
|
||||||
|
```rust
|
||||||
|
// Check shard size and data characteristics
|
||||||
|
let shard_size = data.len().div_ceil(data_shards);
|
||||||
|
println!("Shard size: {} bytes", shard_size);
|
||||||
|
if shard_size >= 1024 {
|
||||||
|
println!("Good candidate for SIMD optimization");
|
||||||
|
} else {
|
||||||
|
println!("Consider using default Erasure mode");
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 问题3: 内存使用异常
|
#### Issue 2: Compilation Errors
|
||||||
**现象**: 内存使用超出预期
|
**Symptom**: SIMD-related compilation errors
|
||||||
**原因**: SIMD 实现的内存对齐要求
|
**Cause**: Platform not supported or missing dependencies
|
||||||
**解决**:
|
**Solution**:
|
||||||
```bash
|
```bash
|
||||||
# 使用纯 Erasure 模式进行对比
|
# Check platform support
|
||||||
|
cargo check --features reed-solomon-simd
|
||||||
|
# If failed, use default mode
|
||||||
|
cargo check
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Issue 3: Abnormal Memory Usage
|
||||||
|
**Symptom**: Memory usage exceeds expectations
|
||||||
|
**Cause**: Memory alignment requirements of SIMD implementation
|
||||||
|
**Solution**:
|
||||||
|
```bash
|
||||||
|
# Use pure Erasure mode for comparison
|
||||||
cargo run --features reed-solomon-erasure
|
cargo run --features reed-solomon-erasure
|
||||||
```
|
```
|
||||||
|
|
||||||
### 调试技巧
|
### Debugging Tips
|
||||||
|
|
||||||
#### 1️⃣ 强制使用特定模式
|
#### 1️⃣ Performance Comparison Testing
|
||||||
```bash
|
```bash
|
||||||
# 测试纯 Erasure 模式性能
|
# Test pure Erasure mode performance
|
||||||
cargo bench --features reed-solomon-erasure
|
cargo bench --features reed-solomon-erasure
|
||||||
|
|
||||||
# 测试混合模式性能(默认)
|
# Test SIMD mode performance
|
||||||
cargo bench
|
cargo bench --features reed-solomon-simd
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 2️⃣ 分析分片大小分布
|
#### 2️⃣ Analyze Data Characteristics
|
||||||
```rust
|
```rust
|
||||||
// 统计你的应用中的分片大小分布
|
// Statistics of data characteristics in your application
|
||||||
let shard_sizes: Vec<usize> = data_samples.iter()
|
let data_sizes: Vec<usize> = data_samples.iter()
|
||||||
.map(|data| data.len().div_ceil(data_shards))
|
.map(|data| data.len())
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let simd_eligible = shard_sizes.iter()
|
let large_data_count = data_sizes.iter()
|
||||||
.filter(|&&size| size >= 512)
|
.filter(|&&size| size >= 1024 * 1024)
|
||||||
.count();
|
.count();
|
||||||
|
|
||||||
println!("SIMD eligible: {}/{} ({}%)",
|
println!("Large data (>1MB): {}/{} ({}%)",
|
||||||
simd_eligible,
|
large_data_count,
|
||||||
shard_sizes.len(),
|
data_sizes.len(),
|
||||||
simd_eligible * 100 / shard_sizes.len()
|
large_data_count * 100 / data_sizes.len()
|
||||||
);
|
);
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 3️⃣ 基准测试对比
|
#### 3️⃣ Benchmark Comparison
|
||||||
```bash
|
```bash
|
||||||
# 生成详细的性能对比报告
|
# Generate detailed performance comparison report
|
||||||
./run_benchmarks.sh comparison
|
./run_benchmarks.sh comparison
|
||||||
|
|
||||||
# 查看 HTML 报告分析性能差异
|
# View HTML report to analyze performance differences
|
||||||
cd target/criterion && python3 -m http.server 8080
|
cd target/criterion && python3 -m http.server 8080
|
||||||
```
|
```
|
||||||
|
|
||||||
## 📈 性能优化建议
|
## 📈 Performance Optimization Recommendations
|
||||||
|
|
||||||
### 应用层优化
|
### Application Layer Optimization
|
||||||
|
|
||||||
#### 1️⃣ 数据分块策略
|
#### 1️⃣ Data Chunking Strategy
|
||||||
```rust
|
```rust
|
||||||
// 针对混合模式优化数据分块
|
// Optimize data chunking for SIMD mode
|
||||||
const OPTIMAL_BLOCK_SIZE: usize = 1024 * 1024; // 1MB
|
const OPTIMAL_BLOCK_SIZE: usize = 1024 * 1024; // 1MB
|
||||||
const MIN_SIMD_BLOCK_SIZE: usize = data_shards * 512; // 确保分片 >= 512B
|
const MIN_EFFICIENT_SIZE: usize = 64 * 1024; // 64KB
|
||||||
|
|
||||||
let block_size = if data.len() < MIN_SIMD_BLOCK_SIZE {
|
let block_size = if data.len() < MIN_EFFICIENT_SIZE {
|
||||||
data.len() // 小数据直接处理,会自动回退
|
data.len() // Small data can consider default mode
|
||||||
} else {
|
} else {
|
||||||
OPTIMAL_BLOCK_SIZE.min(data.len()) // 使用最优块大小
|
OPTIMAL_BLOCK_SIZE.min(data.len()) // Use optimal block size
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 2️⃣ 配置调优
|
#### 2️⃣ Configuration Tuning
|
||||||
```rust
|
```rust
|
||||||
// 根据典型数据大小选择纠删码配置
|
// Choose erasure coding configuration based on typical data size
|
||||||
let (data_shards, parity_shards) = if typical_file_size > 1024 * 1024 {
|
let (data_shards, parity_shards) = if typical_file_size > 1024 * 1024 {
|
||||||
(8, 4) // 大文件:更多并行度,利用 SIMD
|
(8, 4) // Large files: more parallelism, utilize SIMD
|
||||||
} else {
|
} else {
|
||||||
(4, 2) // 小文件:简单配置,减少开销
|
(4, 2) // Small files: simple configuration, reduce overhead
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
### 系统层优化
|
### System Layer Optimization
|
||||||
|
|
||||||
#### 1️⃣ CPU 特性检测
|
#### 1️⃣ CPU Feature Detection
|
||||||
```bash
|
```bash
|
||||||
# 检查 CPU 支持的 SIMD 指令集
|
# Check CPU supported SIMD instruction sets
|
||||||
lscpu | grep -i flags
|
lscpu | grep -i flags
|
||||||
cat /proc/cpuinfo | grep -i flags | head -1
|
cat /proc/cpuinfo | grep -i flags | head -1
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 2️⃣ 内存对齐优化
|
#### 2️⃣ Memory Alignment Optimization
|
||||||
```rust
|
```rust
|
||||||
// 确保数据内存对齐以提升 SIMD 性能
|
// Ensure data memory alignment to improve SIMD performance
|
||||||
use aligned_vec::AlignedVec;
|
use aligned_vec::AlignedVec;
|
||||||
let aligned_data = AlignedVec::<u8, aligned_vec::A64>::from_slice(&data);
|
let aligned_data = AlignedVec::<u8, aligned_vec::A64>::from_slice(&data);
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
💡 **关键结论**:
|
💡 **Key Conclusions**:
|
||||||
- 🎯 **混合模式(默认)是最佳选择**:兼顾性能和兼容性
|
- 🎯 **Pure Erasure mode (default) is the best general choice**: Stable and reliable, suitable for most scenarios
|
||||||
- 🔄 **智能回退机制**:解决了传统 SIMD 模式的兼容性问题
|
- 🚀 **SIMD mode suitable for high-performance scenarios**: Best choice for large data processing
|
||||||
- 📊 **透明优化**:用户无需关心实现细节,系统自动选择最优策略
|
- 📊 **Choose based on data characteristics**: Small data use Erasure, large data consider SIMD
|
||||||
- 🛡️ **零失败风险**:在任何配置下都能正常工作
|
- 🛡️ **Stability priority**: Production environments recommend using default Erasure mode
|
||||||
@@ -0,0 +1,333 @@
|
|||||||
|
# Reed-Solomon 实现对比分析
|
||||||
|
|
||||||
|
## 🔍 问题分析
|
||||||
|
|
||||||
|
随着SIMD模式的优化设计,我们提供了高性能的Reed-Solomon实现。现在系统能够在不同场景下提供最优的性能表现。
|
||||||
|
|
||||||
|
## 📊 实现模式对比
|
||||||
|
|
||||||
|
### 🏛️ 纯 Erasure 模式(默认,推荐)
|
||||||
|
|
||||||
|
**默认配置**: 不指定任何 feature,使用稳定的 reed-solomon-erasure 实现
|
||||||
|
|
||||||
|
**特点**:
|
||||||
|
- ✅ **广泛兼容**: 支持任意分片大小,从字节级到 GB 级
|
||||||
|
- 📈 **稳定性能**: 性能对分片大小不敏感,可预测
|
||||||
|
- 🔧 **生产就绪**: 成熟稳定的实现,已在生产环境广泛使用
|
||||||
|
- 💾 **内存高效**: 优化的内存使用模式
|
||||||
|
- 🎯 **一致性**: 在所有场景下行为完全一致
|
||||||
|
|
||||||
|
**使用场景**:
|
||||||
|
- 大多数生产环境的默认选择
|
||||||
|
- 需要完全一致和可预测的性能行为
|
||||||
|
- 对性能变化敏感的系统
|
||||||
|
- 主要处理小文件或小分片的场景
|
||||||
|
- 需要严格的内存使用控制
|
||||||
|
|
||||||
|
### 🎯 SIMD模式(`reed-solomon-simd` feature)
|
||||||
|
|
||||||
|
**配置**: `--features reed-solomon-simd`
|
||||||
|
|
||||||
|
**特点**:
|
||||||
|
- 🚀 **高性能SIMD**: 使用SIMD指令集进行高性能编码解码
|
||||||
|
- 🎯 **性能导向**: 专注于最大化处理性能
|
||||||
|
- ⚡ **大数据优化**: 适合大数据量处理的高吞吐量场景
|
||||||
|
- 🏎️ **速度优先**: 为性能关键型应用设计
|
||||||
|
|
||||||
|
**使用场景**:
|
||||||
|
- 需要最大化性能的应用场景
|
||||||
|
- 处理大量数据的高吞吐量系统
|
||||||
|
- 对性能要求极高的场景
|
||||||
|
- CPU密集型工作负载
|
||||||
|
|
||||||
|
## 📏 分片大小与性能对比
|
||||||
|
|
||||||
|
不同配置下的性能表现:
|
||||||
|
|
||||||
|
| 数据大小 | 配置 | 分片大小 | 纯 Erasure 模式(默认) | SIMD模式策略 | 性能对比 |
|
||||||
|
|---------|------|----------|------------------------|-------------|----------|
|
||||||
|
| 1KB | 4+2 | 256字节 | Erasure 实现 | SIMD 实现 | SIMD可能更快 |
|
||||||
|
| 1KB | 6+3 | 171字节 | Erasure 实现 | SIMD 实现 | SIMD可能更快 |
|
||||||
|
| 1KB | 8+4 | 128字节 | Erasure 实现 | SIMD 实现 | SIMD可能更快 |
|
||||||
|
| 64KB | 4+2 | 16KB | Erasure 实现 | SIMD 优化 | SIMD模式更快 |
|
||||||
|
| 64KB | 6+3 | 10.7KB | Erasure 实现 | SIMD 优化 | SIMD模式更快 |
|
||||||
|
| 1MB | 4+2 | 256KB | Erasure 实现 | SIMD 优化 | SIMD模式显著更快 |
|
||||||
|
| 16MB | 8+4 | 2MB | Erasure 实现 | SIMD 优化 | SIMD模式大幅领先 |
|
||||||
|
|
||||||
|
## 🎯 基准测试结果解读
|
||||||
|
|
||||||
|
### 纯 Erasure 模式示例(默认) ✅
|
||||||
|
|
||||||
|
```
|
||||||
|
encode_comparison/implementation/1KB_6+3_erasure
|
||||||
|
time: [245.67 ns 256.78 ns 267.89 ns]
|
||||||
|
thrpt: [3.73 GiB/s 3.89 GiB/s 4.07 GiB/s]
|
||||||
|
|
||||||
|
💡 一致的 Erasure 性能 - 所有配置都使用相同实现
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
encode_comparison/implementation/64KB_4+2_erasure
|
||||||
|
time: [2.3456 μs 2.4567 μs 2.5678 μs]
|
||||||
|
thrpt: [23.89 GiB/s 24.65 GiB/s 25.43 GiB/s]
|
||||||
|
|
||||||
|
💡 稳定可靠的性能 - 适合大多数生产场景
|
||||||
|
```
|
||||||
|
|
||||||
|
### SIMD模式成功示例 ✅
|
||||||
|
|
||||||
|
**大分片 SIMD 优化**:
|
||||||
|
```
|
||||||
|
encode_comparison/implementation/64KB_4+2_simd
|
||||||
|
time: [1.2345 μs 1.2567 μs 1.2789 μs]
|
||||||
|
thrpt: [47.89 GiB/s 48.65 GiB/s 49.43 GiB/s]
|
||||||
|
|
||||||
|
💡 使用 SIMD 优化 - 分片大小: 16KB,高性能处理
|
||||||
|
```
|
||||||
|
|
||||||
|
**小分片 SIMD 处理**:
|
||||||
|
```
|
||||||
|
encode_comparison/implementation/1KB_6+3_simd
|
||||||
|
time: [234.56 ns 245.67 ns 256.78 ns]
|
||||||
|
thrpt: [3.89 GiB/s 4.07 GiB/s 4.26 GiB/s]
|
||||||
|
|
||||||
|
💡 SIMD 处理小分片 - 分片大小: 171字节
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🛠️ 使用指南
|
||||||
|
|
||||||
|
### 选择策略
|
||||||
|
|
||||||
|
#### 1️⃣ 推荐:纯 Erasure 模式(默认)
|
||||||
|
```bash
|
||||||
|
# 无需指定 feature,使用默认配置
|
||||||
|
cargo run
|
||||||
|
cargo test
|
||||||
|
cargo bench
|
||||||
|
```
|
||||||
|
|
||||||
|
**适用场景**:
|
||||||
|
- 📊 **一致性要求**: 需要完全可预测的性能行为
|
||||||
|
- 🔬 **生产环境**: 大多数生产场景的最佳选择
|
||||||
|
- 💾 **内存敏感**: 对内存使用模式有严格要求
|
||||||
|
- 🏗️ **稳定可靠**: 成熟稳定的实现
|
||||||
|
|
||||||
|
#### 2️⃣ 高性能需求:SIMD模式
|
||||||
|
```bash
|
||||||
|
# 启用SIMD模式获得最大性能
|
||||||
|
cargo run --features reed-solomon-simd
|
||||||
|
cargo test --features reed-solomon-simd
|
||||||
|
cargo bench --features reed-solomon-simd
|
||||||
|
```
|
||||||
|
|
||||||
|
**适用场景**:
|
||||||
|
- 🎯 **高性能场景**: 处理大量数据需要最大吞吐量
|
||||||
|
- 🚀 **性能优化**: 希望在大数据时获得最佳性能
|
||||||
|
- ⚡ **速度优先**: 对处理速度有极高要求的场景
|
||||||
|
- 🏎️ **计算密集**: CPU密集型工作负载
|
||||||
|
|
||||||
|
### 配置优化建议
|
||||||
|
|
||||||
|
#### 针对数据大小的配置
|
||||||
|
|
||||||
|
**小文件为主** (< 64KB):
|
||||||
|
```toml
|
||||||
|
# 推荐使用默认纯 Erasure 模式
|
||||||
|
# 无需特殊配置,性能稳定可靠
|
||||||
|
```
|
||||||
|
|
||||||
|
**大文件为主** (> 1MB):
|
||||||
|
```toml
|
||||||
|
# 建议启用SIMD模式获得更高性能
|
||||||
|
# features = ["reed-solomon-simd"]
|
||||||
|
```
|
||||||
|
|
||||||
|
**混合场景**:
|
||||||
|
```toml
|
||||||
|
# 默认纯 Erasure 模式适合大多数场景
|
||||||
|
# 如需最大性能可启用: features = ["reed-solomon-simd"]
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 针对纠删码配置的建议
|
||||||
|
|
||||||
|
| 配置 | 小数据 (< 64KB) | 大数据 (> 1MB) | 推荐模式 |
|
||||||
|
|------|----------------|----------------|----------|
|
||||||
|
| 4+2 | 纯 Erasure | 纯 Erasure / SIMD模式 | 纯 Erasure(默认) |
|
||||||
|
| 6+3 | 纯 Erasure | 纯 Erasure / SIMD模式 | 纯 Erasure(默认) |
|
||||||
|
| 8+4 | 纯 Erasure | 纯 Erasure / SIMD模式 | 纯 Erasure(默认) |
|
||||||
|
| 10+5 | 纯 Erasure | 纯 Erasure / SIMD模式 | 纯 Erasure(默认) |
|
||||||
|
|
||||||
|
### 生产环境部署建议
|
||||||
|
|
||||||
|
#### 1️⃣ 默认部署策略
|
||||||
|
```bash
|
||||||
|
# 生产环境推荐配置:使用纯 Erasure 模式(默认)
|
||||||
|
cargo build --release
|
||||||
|
```
|
||||||
|
|
||||||
|
**优势**:
|
||||||
|
- ✅ 最大兼容性:处理任意大小数据
|
||||||
|
- ✅ 稳定可靠:成熟的实现,行为可预测
|
||||||
|
- ✅ 零配置:无需复杂的性能调优
|
||||||
|
- ✅ 内存高效:优化的内存使用模式
|
||||||
|
|
||||||
|
#### 2️⃣ 高性能部署策略
|
||||||
|
```bash
|
||||||
|
# 高性能场景:启用SIMD模式
|
||||||
|
cargo build --release --features reed-solomon-simd
|
||||||
|
```
|
||||||
|
|
||||||
|
**优势**:
|
||||||
|
- ✅ 最优性能:SIMD指令集优化
|
||||||
|
- ✅ 高吞吐量:适合大数据处理
|
||||||
|
- ✅ 性能导向:专注于最大化处理速度
|
||||||
|
- ✅ 现代硬件:充分利用现代CPU特性
|
||||||
|
|
||||||
|
#### 2️⃣ 监控和调优
|
||||||
|
```rust
|
||||||
|
// 根据具体场景选择合适的实现
|
||||||
|
match data_size {
|
||||||
|
size if size > 1024 * 1024 => {
|
||||||
|
// 大数据:考虑使用SIMD模式
|
||||||
|
println!("Large data detected, SIMD mode recommended");
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
// 一般情况:使用默认Erasure模式
|
||||||
|
println!("Using default Erasure mode");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 3️⃣ 性能监控指标
|
||||||
|
- **吞吐量监控**: 监控编码/解码的数据处理速率
|
||||||
|
- **延迟分析**: 分析不同数据大小的处理延迟
|
||||||
|
- **CPU使用率**: 观察SIMD指令的CPU利用效率
|
||||||
|
- **内存使用**: 监控不同实现的内存分配模式
|
||||||
|
|
||||||
|
## 🔧 故障排除
|
||||||
|
|
||||||
|
### 性能问题诊断
|
||||||
|
|
||||||
|
#### 问题1: 性能不符合预期
|
||||||
|
**现象**: SIMD模式性能提升不明显
|
||||||
|
**原因**: 可能数据大小不适合SIMD优化
|
||||||
|
**解决**:
|
||||||
|
```rust
|
||||||
|
// 检查分片大小和数据特征
|
||||||
|
let shard_size = data.len().div_ceil(data_shards);
|
||||||
|
println!("Shard size: {} bytes", shard_size);
|
||||||
|
if shard_size >= 1024 {
|
||||||
|
println!("Good candidate for SIMD optimization");
|
||||||
|
} else {
|
||||||
|
println!("Consider using default Erasure mode");
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 问题2: 编译错误
|
||||||
|
**现象**: SIMD相关的编译错误
|
||||||
|
**原因**: 平台不支持或依赖缺失
|
||||||
|
**解决**:
|
||||||
|
```bash
|
||||||
|
# 检查平台支持
|
||||||
|
cargo check --features reed-solomon-simd
|
||||||
|
# 如果失败,使用默认模式
|
||||||
|
cargo check
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 问题3: 内存使用异常
|
||||||
|
**现象**: 内存使用超出预期
|
||||||
|
**原因**: SIMD实现的内存对齐要求
|
||||||
|
**解决**:
|
||||||
|
```bash
|
||||||
|
# 使用纯 Erasure 模式进行对比
|
||||||
|
cargo run --features reed-solomon-erasure
|
||||||
|
```
|
||||||
|
|
||||||
|
### 调试技巧
|
||||||
|
|
||||||
|
#### 1️⃣ 性能对比测试
|
||||||
|
```bash
|
||||||
|
# 测试纯 Erasure 模式性能
|
||||||
|
cargo bench --features reed-solomon-erasure
|
||||||
|
|
||||||
|
# 测试SIMD模式性能
|
||||||
|
cargo bench --features reed-solomon-simd
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 2️⃣ 分析数据特征
|
||||||
|
```rust
|
||||||
|
// 统计你的应用中的数据特征
|
||||||
|
let data_sizes: Vec<usize> = data_samples.iter()
|
||||||
|
.map(|data| data.len())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let large_data_count = data_sizes.iter()
|
||||||
|
.filter(|&&size| size >= 1024 * 1024)
|
||||||
|
.count();
|
||||||
|
|
||||||
|
println!("Large data (>1MB): {}/{} ({}%)",
|
||||||
|
large_data_count,
|
||||||
|
data_sizes.len(),
|
||||||
|
large_data_count * 100 / data_sizes.len()
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 3️⃣ 基准测试对比
|
||||||
|
```bash
|
||||||
|
# 生成详细的性能对比报告
|
||||||
|
./run_benchmarks.sh comparison
|
||||||
|
|
||||||
|
# 查看 HTML 报告分析性能差异
|
||||||
|
cd target/criterion && python3 -m http.server 8080
|
||||||
|
```
|
||||||
|
|
||||||
|
## 📈 性能优化建议
|
||||||
|
|
||||||
|
### 应用层优化
|
||||||
|
|
||||||
|
#### 1️⃣ 数据分块策略
|
||||||
|
```rust
|
||||||
|
// 针对SIMD模式优化数据分块
|
||||||
|
const OPTIMAL_BLOCK_SIZE: usize = 1024 * 1024; // 1MB
|
||||||
|
const MIN_EFFICIENT_SIZE: usize = 64 * 1024; // 64KB
|
||||||
|
|
||||||
|
let block_size = if data.len() < MIN_EFFICIENT_SIZE {
|
||||||
|
data.len() // 小数据可以考虑默认模式
|
||||||
|
} else {
|
||||||
|
OPTIMAL_BLOCK_SIZE.min(data.len()) // 使用最优块大小
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 2️⃣ 配置调优
|
||||||
|
```rust
|
||||||
|
// 根据典型数据大小选择纠删码配置
|
||||||
|
let (data_shards, parity_shards) = if typical_file_size > 1024 * 1024 {
|
||||||
|
(8, 4) // 大文件:更多并行度,利用 SIMD
|
||||||
|
} else {
|
||||||
|
(4, 2) // 小文件:简单配置,减少开销
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
### 系统层优化
|
||||||
|
|
||||||
|
#### 1️⃣ CPU 特性检测
|
||||||
|
```bash
|
||||||
|
# 检查 CPU 支持的 SIMD 指令集
|
||||||
|
lscpu | grep -i flags
|
||||||
|
cat /proc/cpuinfo | grep -i flags | head -1
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 2️⃣ 内存对齐优化
|
||||||
|
```rust
|
||||||
|
// 确保数据内存对齐以提升 SIMD 性能
|
||||||
|
use aligned_vec::AlignedVec;
|
||||||
|
let aligned_data = AlignedVec::<u8, aligned_vec::A64>::from_slice(&data);
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
💡 **关键结论**:
|
||||||
|
- 🎯 **纯Erasure模式(默认)是最佳通用选择**:稳定可靠,适合大多数场景
|
||||||
|
- 🚀 **SIMD模式适合高性能场景**:大数据处理的最佳选择
|
||||||
|
- 📊 **根据数据特征选择**:小数据用Erasure,大数据考虑SIMD
|
||||||
|
- 🛡️ **稳定性优先**:生产环境建议使用默认Erasure模式
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
//!
|
//!
|
||||||
//! This benchmark compares the performance of different Reed-Solomon implementations:
|
//! This benchmark compares the performance of different Reed-Solomon implementations:
|
||||||
//! - Default (Pure erasure): Stable reed-solomon-erasure implementation
|
//! - Default (Pure erasure): Stable reed-solomon-erasure implementation
|
||||||
//! - `reed-solomon-simd` feature: Hybrid mode with SIMD optimization and erasure fallback
|
//! - `reed-solomon-simd` feature: SIMD mode with optimized performance
|
||||||
//!
|
//!
|
||||||
//! ## Running Benchmarks
|
//! ## Running Benchmarks
|
||||||
//!
|
//!
|
||||||
|
|||||||
+28
-25
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
# Reed-Solomon 实现性能比较脚本
|
# Reed-Solomon 实现性能比较脚本
|
||||||
#
|
#
|
||||||
# 这个脚本将运行不同的基准测试来比较混合模式和纯Erasure模式的性能
|
# 这个脚本将运行不同的基准测试来比较SIMD模式和纯Erasure模式的性能
|
||||||
#
|
#
|
||||||
# 使用方法:
|
# 使用方法:
|
||||||
# ./run_benchmarks.sh [quick|full|comparison]
|
# ./run_benchmarks.sh [quick|full|comparison]
|
||||||
@@ -74,15 +74,16 @@ run_erasure_benchmark() {
|
|||||||
print_success "纯 Erasure 模式基准测试完成"
|
print_success "纯 Erasure 模式基准测试完成"
|
||||||
}
|
}
|
||||||
|
|
||||||
# 运行混合模式基准测试(默认)
|
# 运行SIMD模式基准测试
|
||||||
run_hybrid_benchmark() {
|
run_simd_benchmark() {
|
||||||
print_info "🎯 开始运行混合模式基准测试(默认)..."
|
print_info "🎯 开始运行SIMD模式基准测试..."
|
||||||
echo "================================================"
|
echo "================================================"
|
||||||
|
|
||||||
cargo bench --bench comparison_benchmark \
|
cargo bench --bench comparison_benchmark \
|
||||||
-- --save-baseline hybrid_baseline
|
--features reed-solomon-simd \
|
||||||
|
-- --save-baseline simd_baseline
|
||||||
|
|
||||||
print_success "混合模式基准测试完成"
|
print_success "SIMD模式基准测试完成"
|
||||||
}
|
}
|
||||||
|
|
||||||
# 运行完整的基准测试套件
|
# 运行完整的基准测试套件
|
||||||
@@ -90,7 +91,7 @@ run_full_benchmark() {
|
|||||||
print_info "🚀 开始运行完整基准测试套件..."
|
print_info "🚀 开始运行完整基准测试套件..."
|
||||||
echo "================================================"
|
echo "================================================"
|
||||||
|
|
||||||
# 运行详细的基准测试(使用默认混合模式)
|
# 运行详细的基准测试(使用默认纯Erasure模式)
|
||||||
cargo bench --bench erasure_benchmark
|
cargo bench --bench erasure_benchmark
|
||||||
|
|
||||||
print_success "完整基准测试套件完成"
|
print_success "完整基准测试套件完成"
|
||||||
@@ -106,8 +107,9 @@ run_comparison_benchmark() {
|
|||||||
--features reed-solomon-erasure \
|
--features reed-solomon-erasure \
|
||||||
-- --save-baseline erasure_baseline
|
-- --save-baseline erasure_baseline
|
||||||
|
|
||||||
print_info "步骤 2: 测试混合模式并与 Erasure 模式对比..."
|
print_info "步骤 2: 测试SIMD模式并与 Erasure 模式对比..."
|
||||||
cargo bench --bench comparison_benchmark \
|
cargo bench --bench comparison_benchmark \
|
||||||
|
--features reed-solomon-simd \
|
||||||
-- --baseline erasure_baseline
|
-- --baseline erasure_baseline
|
||||||
|
|
||||||
print_success "性能对比测试完成"
|
print_success "性能对比测试完成"
|
||||||
@@ -141,8 +143,9 @@ run_quick_test() {
|
|||||||
--features reed-solomon-erasure \
|
--features reed-solomon-erasure \
|
||||||
-- encode_comparison --quick
|
-- encode_comparison --quick
|
||||||
|
|
||||||
print_info "测试混合模式(默认)..."
|
print_info "测试SIMD模式..."
|
||||||
cargo bench --bench comparison_benchmark \
|
cargo bench --bench comparison_benchmark \
|
||||||
|
--features reed-solomon-simd \
|
||||||
-- encode_comparison --quick
|
-- encode_comparison --quick
|
||||||
|
|
||||||
print_success "快速测试完成"
|
print_success "快速测试完成"
|
||||||
@@ -153,31 +156,31 @@ show_help() {
|
|||||||
echo "Reed-Solomon 性能基准测试脚本"
|
echo "Reed-Solomon 性能基准测试脚本"
|
||||||
echo ""
|
echo ""
|
||||||
echo "实现模式:"
|
echo "实现模式:"
|
||||||
echo " 🎯 混合模式(默认) - SIMD + Erasure 智能回退,推荐使用"
|
echo " 🏛️ 纯 Erasure 模式(默认)- 稳定兼容的 reed-solomon-erasure 实现"
|
||||||
echo " 🏛️ 纯 Erasure 模式 - 稳定兼容的 reed-solomon-erasure 实现"
|
echo " 🎯 SIMD模式 - 高性能SIMD优化实现"
|
||||||
echo ""
|
echo ""
|
||||||
echo "使用方法:"
|
echo "使用方法:"
|
||||||
echo " $0 [command]"
|
echo " $0 [command]"
|
||||||
echo ""
|
echo ""
|
||||||
echo "命令:"
|
echo "命令:"
|
||||||
echo " quick 运行快速性能测试"
|
echo " quick 运行快速性能测试"
|
||||||
echo " full 运行完整基准测试套件(混合模式)"
|
echo " full 运行完整基准测试套件(默认Erasure模式)"
|
||||||
echo " comparison 运行详细的实现模式对比测试"
|
echo " comparison 运行详细的实现模式对比测试"
|
||||||
echo " erasure 只测试纯 Erasure 模式"
|
echo " erasure 只测试纯 Erasure 模式"
|
||||||
echo " hybrid 只测试混合模式(默认行为)"
|
echo " simd 只测试SIMD模式"
|
||||||
echo " clean 清理测试结果"
|
echo " clean 清理测试结果"
|
||||||
echo " help 显示此帮助信息"
|
echo " help 显示此帮助信息"
|
||||||
echo ""
|
echo ""
|
||||||
echo "示例:"
|
echo "示例:"
|
||||||
echo " $0 quick # 快速测试两种模式"
|
echo " $0 quick # 快速测试两种模式"
|
||||||
echo " $0 comparison # 详细对比测试"
|
echo " $0 comparison # 详细对比测试"
|
||||||
echo " $0 full # 完整测试套件(混合模式)"
|
echo " $0 full # 完整测试套件(默认Erasure模式)"
|
||||||
echo " $0 hybrid # 只测试混合模式"
|
echo " $0 simd # 只测试SIMD模式"
|
||||||
echo " $0 erasure # 只测试纯 Erasure 模式"
|
echo " $0 erasure # 只测试纯 Erasure 模式"
|
||||||
echo ""
|
echo ""
|
||||||
echo "模式说明:"
|
echo "模式说明:"
|
||||||
echo " 混合模式: 大分片(≥512B)使用SIMD优化,小分片自动回退到Erasure"
|
echo " Erasure模式: 使用reed-solomon-erasure实现,稳定可靠"
|
||||||
echo " Erasure模式: 所有情况都使用reed-solomon-erasure实现"
|
echo " SIMD模式: 使用reed-solomon-simd实现,高性能优化"
|
||||||
}
|
}
|
||||||
|
|
||||||
# 显示测试配置信息
|
# 显示测试配置信息
|
||||||
@@ -193,16 +196,16 @@ show_test_info() {
|
|||||||
if [ -f "/proc/cpuinfo" ]; then
|
if [ -f "/proc/cpuinfo" ]; then
|
||||||
echo " - CPU 型号: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs)"
|
echo " - CPU 型号: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs)"
|
||||||
if grep -q "avx2" /proc/cpuinfo; then
|
if grep -q "avx2" /proc/cpuinfo; then
|
||||||
echo " - SIMD 支持: AVX2 ✅ (混合模式将利用SIMD优化)"
|
echo " - SIMD 支持: AVX2 ✅ (SIMD模式将利用SIMD优化)"
|
||||||
elif grep -q "sse4" /proc/cpuinfo; then
|
elif grep -q "sse4" /proc/cpuinfo; then
|
||||||
echo " - SIMD 支持: SSE4 ✅ (混合模式将利用SIMD优化)"
|
echo " - SIMD 支持: SSE4 ✅ (SIMD模式将利用SIMD优化)"
|
||||||
else
|
else
|
||||||
echo " - SIMD 支持: 未检测到高级 SIMD 特性 (混合模式将主要使用Erasure)"
|
echo " - SIMD 支持: 未检测到高级 SIMD 特性"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo " - 默认模式: 混合模式 (SIMD + Erasure 智能回退)"
|
echo " - 默认模式: 纯Erasure模式 (稳定可靠)"
|
||||||
echo " - 回退阈值: 512字节分片大小"
|
echo " - 高性能模式: SIMD模式 (性能优化)"
|
||||||
echo ""
|
echo ""
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -234,9 +237,9 @@ main() {
|
|||||||
run_erasure_benchmark
|
run_erasure_benchmark
|
||||||
generate_comparison_report
|
generate_comparison_report
|
||||||
;;
|
;;
|
||||||
"hybrid")
|
"simd")
|
||||||
cleanup
|
cleanup
|
||||||
run_hybrid_benchmark
|
run_simd_benchmark
|
||||||
generate_comparison_report
|
generate_comparison_report
|
||||||
;;
|
;;
|
||||||
"clean")
|
"clean")
|
||||||
@@ -254,7 +257,7 @@ main() {
|
|||||||
esac
|
esac
|
||||||
|
|
||||||
print_success "✨ 基准测试执行完成!"
|
print_success "✨ 基准测试执行完成!"
|
||||||
print_info "💡 提示: 推荐使用混合模式(默认),它能自动在SIMD和Erasure之间智能选择"
|
print_info "💡 提示: 推荐使用默认的纯Erasure模式,对于高性能需求可考虑SIMD模式"
|
||||||
}
|
}
|
||||||
|
|
||||||
# 如果直接运行此脚本,调用主函数
|
# 如果直接运行此脚本,调用主函数
|
||||||
|
|||||||
+167
-841
File diff suppressed because it is too large
Load Diff
@@ -38,13 +38,13 @@ use crate::utils::path::{
|
|||||||
path_join, path_join_buf,
|
path_join, path_join_buf,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use crate::erasure_coding::bitrot_verify;
|
||||||
use common::defer;
|
use common::defer;
|
||||||
use path_absolutize::Absolutize;
|
use path_absolutize::Absolutize;
|
||||||
use rustfs_filemeta::{
|
use rustfs_filemeta::{
|
||||||
Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, Opts, RawFileInfo, UpdateFn, get_file_info,
|
Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, Opts, RawFileInfo, UpdateFn, get_file_info,
|
||||||
read_xl_meta_no_data,
|
read_xl_meta_no_data,
|
||||||
};
|
};
|
||||||
use rustfs_rio::bitrot_verify;
|
|
||||||
use rustfs_utils::HashAlgorithm;
|
use rustfs_utils::HashAlgorithm;
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::fmt::Debug;
|
use std::fmt::Debug;
|
||||||
|
|||||||
@@ -0,0 +1,458 @@
|
|||||||
|
use pin_project_lite::pin_project;
|
||||||
|
use rustfs_utils::{HashAlgorithm, read_full, write_all};
|
||||||
|
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite};
|
||||||
|
|
||||||
|
pin_project! {
|
||||||
|
/// BitrotReader reads (hash+data) blocks from an async reader and verifies hash integrity.
|
||||||
|
pub struct BitrotReader<R> {
|
||||||
|
#[pin]
|
||||||
|
inner: R,
|
||||||
|
hash_algo: HashAlgorithm,
|
||||||
|
shard_size: usize,
|
||||||
|
buf: Vec<u8>,
|
||||||
|
hash_buf: Vec<u8>,
|
||||||
|
hash_read: usize,
|
||||||
|
data_buf: Vec<u8>,
|
||||||
|
data_read: usize,
|
||||||
|
hash_checked: bool,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<R> BitrotReader<R>
|
||||||
|
where
|
||||||
|
R: AsyncRead + Unpin + Send + Sync,
|
||||||
|
{
|
||||||
|
/// Create a new BitrotReader.
|
||||||
|
pub fn new(inner: R, shard_size: usize, algo: HashAlgorithm) -> Self {
|
||||||
|
let hash_size = algo.size();
|
||||||
|
Self {
|
||||||
|
inner,
|
||||||
|
hash_algo: algo,
|
||||||
|
shard_size,
|
||||||
|
buf: Vec::new(),
|
||||||
|
hash_buf: vec![0u8; hash_size],
|
||||||
|
hash_read: 0,
|
||||||
|
data_buf: Vec::new(),
|
||||||
|
data_read: 0,
|
||||||
|
hash_checked: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read a single (hash+data) block, verify hash, and return the number of bytes read into `out`.
|
||||||
|
/// Returns an error if hash verification fails or data exceeds shard_size.
|
||||||
|
pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
|
||||||
|
if out.len() > self.shard_size {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidInput,
|
||||||
|
format!("data size {} exceeds shard size {}", out.len(), self.shard_size),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let hash_size = self.hash_algo.size();
|
||||||
|
// Read hash
|
||||||
|
let mut hash_buf = vec![0u8; hash_size];
|
||||||
|
if hash_size > 0 {
|
||||||
|
self.inner.read_exact(&mut hash_buf).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let data_len = read_full(&mut self.inner, out).await?;
|
||||||
|
|
||||||
|
// // Read data
|
||||||
|
// let mut data_len = 0;
|
||||||
|
// while data_len < out.len() {
|
||||||
|
// let n = self.inner.read(&mut out[data_len..]).await?;
|
||||||
|
// if n == 0 {
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// data_len += n;
|
||||||
|
// // Only read up to one shard_size block
|
||||||
|
// if data_len >= self.shard_size {
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
if hash_size > 0 {
|
||||||
|
let actual_hash = self.hash_algo.hash_encode(&out[..data_len]);
|
||||||
|
if actual_hash != hash_buf {
|
||||||
|
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(data_len)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pin_project! {
|
||||||
|
/// BitrotWriter writes (hash+data) blocks to an async writer.
|
||||||
|
pub struct BitrotWriter<W> {
|
||||||
|
#[pin]
|
||||||
|
inner: W,
|
||||||
|
hash_algo: HashAlgorithm,
|
||||||
|
shard_size: usize,
|
||||||
|
buf: Vec<u8>,
|
||||||
|
finished: bool,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<W> BitrotWriter<W>
|
||||||
|
where
|
||||||
|
W: AsyncWrite + Unpin + Send + Sync,
|
||||||
|
{
|
||||||
|
/// Create a new BitrotWriter.
|
||||||
|
pub fn new(inner: W, shard_size: usize, algo: HashAlgorithm) -> Self {
|
||||||
|
let hash_algo = algo;
|
||||||
|
Self {
|
||||||
|
inner,
|
||||||
|
hash_algo,
|
||||||
|
shard_size,
|
||||||
|
buf: Vec::new(),
|
||||||
|
finished: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn into_inner(self) -> W {
|
||||||
|
self.inner
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write a (hash+data) block. Returns the number of data bytes written.
|
||||||
|
/// Returns an error if called after a short write or if data exceeds shard_size.
|
||||||
|
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||||
|
if buf.is_empty() {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.finished {
|
||||||
|
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "bitrot writer already finished"));
|
||||||
|
}
|
||||||
|
|
||||||
|
if buf.len() > self.shard_size {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidInput,
|
||||||
|
format!("data size {} exceeds shard size {}", buf.len(), self.shard_size),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if buf.len() < self.shard_size {
|
||||||
|
self.finished = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let hash_algo = &self.hash_algo;
|
||||||
|
|
||||||
|
if hash_algo.size() > 0 {
|
||||||
|
let hash = hash_algo.hash_encode(buf);
|
||||||
|
self.buf.extend_from_slice(&hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.buf.extend_from_slice(buf);
|
||||||
|
|
||||||
|
// Write hash+data in one call
|
||||||
|
let mut n = write_all(&mut self.inner, &self.buf).await?;
|
||||||
|
|
||||||
|
if n < hash_algo.size() {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::WriteZero,
|
||||||
|
"short write: not enough bytes written",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
n -= hash_algo.size();
|
||||||
|
|
||||||
|
self.buf.clear();
|
||||||
|
|
||||||
|
Ok(n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize {
|
||||||
|
if algo != HashAlgorithm::HighwayHash256S {
|
||||||
|
return size;
|
||||||
|
}
|
||||||
|
size.div_ceil(shard_size) * algo.size() + size
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
|
||||||
|
mut r: R,
|
||||||
|
want_size: usize,
|
||||||
|
part_size: usize,
|
||||||
|
algo: HashAlgorithm,
|
||||||
|
_want: Vec<u8>,
|
||||||
|
mut shard_size: usize,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
let mut hash_buf = vec![0; algo.size()];
|
||||||
|
let mut left = want_size;
|
||||||
|
|
||||||
|
if left != bitrot_shard_file_size(part_size, shard_size, algo.clone()) {
|
||||||
|
return Err(std::io::Error::other("bitrot shard file size mismatch"));
|
||||||
|
}
|
||||||
|
|
||||||
|
while left > 0 {
|
||||||
|
let n = r.read_exact(&mut hash_buf).await?;
|
||||||
|
left -= n;
|
||||||
|
|
||||||
|
if left < shard_size {
|
||||||
|
shard_size = left;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut buf = vec![0; shard_size];
|
||||||
|
let read = r.read_exact(&mut buf).await?;
|
||||||
|
|
||||||
|
let actual_hash = algo.hash_encode(&buf);
|
||||||
|
if actual_hash != hash_buf[0..n] {
|
||||||
|
return Err(std::io::Error::other("bitrot hash mismatch"));
|
||||||
|
}
|
||||||
|
|
||||||
|
left -= read;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Custom writer enum that supports inline buffer storage
|
||||||
|
pub enum CustomWriter {
|
||||||
|
/// Inline buffer writer - stores data in memory
|
||||||
|
InlineBuffer(Vec<u8>),
|
||||||
|
/// Disk-based writer using tokio file
|
||||||
|
Other(Box<dyn AsyncWrite + Unpin + Send + Sync>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CustomWriter {
|
||||||
|
/// Create a new inline buffer writer
|
||||||
|
pub fn new_inline_buffer() -> Self {
|
||||||
|
Self::InlineBuffer(Vec::new())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a new disk writer from any AsyncWrite implementation
|
||||||
|
pub fn new_tokio_writer<W>(writer: W) -> Self
|
||||||
|
where
|
||||||
|
W: AsyncWrite + Unpin + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
Self::Other(Box::new(writer))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get the inline buffer data if this is an inline buffer writer
|
||||||
|
pub fn get_inline_data(&self) -> Option<&[u8]> {
|
||||||
|
match self {
|
||||||
|
Self::InlineBuffer(data) => Some(data),
|
||||||
|
Self::Other(_) => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract the inline buffer data, consuming the writer
|
||||||
|
pub fn into_inline_data(self) -> Option<Vec<u8>> {
|
||||||
|
match self {
|
||||||
|
Self::InlineBuffer(data) => Some(data),
|
||||||
|
Self::Other(_) => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncWrite for CustomWriter {
|
||||||
|
fn poll_write(
|
||||||
|
self: std::pin::Pin<&mut Self>,
|
||||||
|
cx: &mut std::task::Context<'_>,
|
||||||
|
buf: &[u8],
|
||||||
|
) -> std::task::Poll<std::io::Result<usize>> {
|
||||||
|
match self.get_mut() {
|
||||||
|
Self::InlineBuffer(data) => {
|
||||||
|
data.extend_from_slice(buf);
|
||||||
|
std::task::Poll::Ready(Ok(buf.len()))
|
||||||
|
}
|
||||||
|
Self::Other(writer) => {
|
||||||
|
let pinned_writer = std::pin::Pin::new(writer.as_mut());
|
||||||
|
pinned_writer.poll_write(cx, buf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_flush(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||||
|
match self.get_mut() {
|
||||||
|
Self::InlineBuffer(_) => std::task::Poll::Ready(Ok(())),
|
||||||
|
Self::Other(writer) => {
|
||||||
|
let pinned_writer = std::pin::Pin::new(writer.as_mut());
|
||||||
|
pinned_writer.poll_flush(cx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_shutdown(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||||
|
match self.get_mut() {
|
||||||
|
Self::InlineBuffer(_) => std::task::Poll::Ready(Ok(())),
|
||||||
|
Self::Other(writer) => {
|
||||||
|
let pinned_writer = std::pin::Pin::new(writer.as_mut());
|
||||||
|
pinned_writer.poll_shutdown(cx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wrapper around BitrotWriter that uses our custom writer
|
||||||
|
pub struct BitrotWriterWrapper {
|
||||||
|
bitrot_writer: BitrotWriter<CustomWriter>,
|
||||||
|
writer_type: WriterType,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Enum to track the type of writer we're using
|
||||||
|
enum WriterType {
|
||||||
|
InlineBuffer,
|
||||||
|
Other,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for BitrotWriterWrapper {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("BitrotWriterWrapper")
|
||||||
|
.field(
|
||||||
|
"writer_type",
|
||||||
|
&match self.writer_type {
|
||||||
|
WriterType::InlineBuffer => "InlineBuffer",
|
||||||
|
WriterType::Other => "Other",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BitrotWriterWrapper {
|
||||||
|
/// Create a new BitrotWriterWrapper with custom writer
|
||||||
|
pub fn new(writer: CustomWriter, shard_size: usize, checksum_algo: HashAlgorithm) -> Self {
|
||||||
|
let writer_type = match &writer {
|
||||||
|
CustomWriter::InlineBuffer(_) => WriterType::InlineBuffer,
|
||||||
|
CustomWriter::Other(_) => WriterType::Other,
|
||||||
|
};
|
||||||
|
|
||||||
|
Self {
|
||||||
|
bitrot_writer: BitrotWriter::new(writer, shard_size, checksum_algo),
|
||||||
|
writer_type,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write data to the bitrot writer
|
||||||
|
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||||
|
self.bitrot_writer.write(buf).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract the inline buffer data, consuming the wrapper
|
||||||
|
pub fn into_inline_data(self) -> Option<Vec<u8>> {
|
||||||
|
match self.writer_type {
|
||||||
|
WriterType::InlineBuffer => {
|
||||||
|
let writer = self.bitrot_writer.into_inner();
|
||||||
|
writer.into_inline_data()
|
||||||
|
}
|
||||||
|
WriterType::Other => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
|
||||||
|
use super::BitrotReader;
|
||||||
|
use super::BitrotWriter;
|
||||||
|
use rustfs_utils::HashAlgorithm;
|
||||||
|
use std::io::Cursor;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_bitrot_read_write_ok() {
|
||||||
|
let data = b"hello world! this is a test shard.";
|
||||||
|
let data_size = data.len();
|
||||||
|
let shard_size = 8;
|
||||||
|
|
||||||
|
let buf: Vec<u8> = Vec::new();
|
||||||
|
let writer = Cursor::new(buf);
|
||||||
|
let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256);
|
||||||
|
|
||||||
|
let mut n = 0;
|
||||||
|
for chunk in data.chunks(shard_size) {
|
||||||
|
n += bitrot_writer.write(chunk).await.unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(n, data.len());
|
||||||
|
|
||||||
|
// 读
|
||||||
|
let reader = bitrot_writer.into_inner();
|
||||||
|
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut n = 0;
|
||||||
|
while n < data_size {
|
||||||
|
let mut buf = vec![0u8; shard_size];
|
||||||
|
let m = bitrot_reader.read(&mut buf).await.unwrap();
|
||||||
|
assert_eq!(&buf[..m], &data[n..n + m]);
|
||||||
|
|
||||||
|
out.extend_from_slice(&buf[..m]);
|
||||||
|
n += m;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(n, data_size);
|
||||||
|
assert_eq!(data, &out[..]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_bitrot_read_hash_mismatch() {
|
||||||
|
let data = b"test data for bitrot";
|
||||||
|
let data_size = data.len();
|
||||||
|
let shard_size = 8;
|
||||||
|
let buf: Vec<u8> = Vec::new();
|
||||||
|
let writer = Cursor::new(buf);
|
||||||
|
let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256);
|
||||||
|
for chunk in data.chunks(shard_size) {
|
||||||
|
let _ = bitrot_writer.write(chunk).await.unwrap();
|
||||||
|
}
|
||||||
|
let mut written = bitrot_writer.into_inner().into_inner();
|
||||||
|
// change the last byte to make hash mismatch
|
||||||
|
let pos = written.len() - 1;
|
||||||
|
written[pos] ^= 0xFF;
|
||||||
|
let reader = Cursor::new(written);
|
||||||
|
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
|
||||||
|
|
||||||
|
let count = data_size.div_ceil(shard_size);
|
||||||
|
|
||||||
|
let mut idx = 0;
|
||||||
|
let mut n = 0;
|
||||||
|
while n < data_size {
|
||||||
|
let mut buf = vec![0u8; shard_size];
|
||||||
|
let res = bitrot_reader.read(&mut buf).await;
|
||||||
|
|
||||||
|
if idx == count - 1 {
|
||||||
|
// 最后一个块,应该返回错误
|
||||||
|
assert!(res.is_err());
|
||||||
|
assert_eq!(res.unwrap_err().kind(), std::io::ErrorKind::InvalidData);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
let m = res.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(&buf[..m], &data[n..n + m]);
|
||||||
|
|
||||||
|
n += m;
|
||||||
|
idx += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_bitrot_read_write_none_hash() {
|
||||||
|
let data = b"bitrot none hash test data!";
|
||||||
|
let data_size = data.len();
|
||||||
|
let shard_size = 8;
|
||||||
|
|
||||||
|
let buf: Vec<u8> = Vec::new();
|
||||||
|
let writer = Cursor::new(buf);
|
||||||
|
let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::None);
|
||||||
|
|
||||||
|
let mut n = 0;
|
||||||
|
for chunk in data.chunks(shard_size) {
|
||||||
|
n += bitrot_writer.write(chunk).await.unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(n, data.len());
|
||||||
|
|
||||||
|
let reader = bitrot_writer.into_inner();
|
||||||
|
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::None);
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut n = 0;
|
||||||
|
while n < data_size {
|
||||||
|
let mut buf = vec![0u8; shard_size];
|
||||||
|
let m = bitrot_reader.read(&mut buf).await.unwrap();
|
||||||
|
assert_eq!(&buf[..m], &data[n..n + m]);
|
||||||
|
out.extend_from_slice(&buf[..m]);
|
||||||
|
n += m;
|
||||||
|
}
|
||||||
|
assert_eq!(n, data_size);
|
||||||
|
assert_eq!(data, &out[..]);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,18 +1,20 @@
|
|||||||
|
use super::BitrotReader;
|
||||||
use super::Erasure;
|
use super::Erasure;
|
||||||
use crate::disk::error::Error;
|
use crate::disk::error::Error;
|
||||||
use crate::disk::error_reduce::reduce_errs;
|
use crate::disk::error_reduce::reduce_errs;
|
||||||
use futures::future::join_all;
|
use futures::future::join_all;
|
||||||
use pin_project_lite::pin_project;
|
use pin_project_lite::pin_project;
|
||||||
use rustfs_rio::BitrotReader;
|
|
||||||
use std::io;
|
use std::io;
|
||||||
use std::io::ErrorKind;
|
use std::io::ErrorKind;
|
||||||
|
use tokio::io::AsyncRead;
|
||||||
|
use tokio::io::AsyncWrite;
|
||||||
use tokio::io::AsyncWriteExt;
|
use tokio::io::AsyncWriteExt;
|
||||||
use tracing::error;
|
use tracing::error;
|
||||||
|
|
||||||
pin_project! {
|
pin_project! {
|
||||||
pub(crate) struct ParallelReader {
|
pub(crate) struct ParallelReader<R> {
|
||||||
#[pin]
|
#[pin]
|
||||||
readers: Vec<Option<BitrotReader>>,
|
readers: Vec<Option<BitrotReader<R>>>,
|
||||||
offset: usize,
|
offset: usize,
|
||||||
shard_size: usize,
|
shard_size: usize,
|
||||||
shard_file_size: usize,
|
shard_file_size: usize,
|
||||||
@@ -21,9 +23,12 @@ pub(crate) struct ParallelReader {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ParallelReader {
|
impl<R> ParallelReader<R>
|
||||||
|
where
|
||||||
|
R: AsyncRead + Unpin + Send + Sync,
|
||||||
|
{
|
||||||
// readers传入前应处理disk错误,确保每个reader达到可用数量的BitrotReader
|
// readers传入前应处理disk错误,确保每个reader达到可用数量的BitrotReader
|
||||||
pub fn new(readers: Vec<Option<BitrotReader>>, e: Erasure, offset: usize, total_length: usize) -> Self {
|
pub fn new(readers: Vec<Option<BitrotReader<R>>>, e: Erasure, offset: usize, total_length: usize) -> Self {
|
||||||
let shard_size = e.shard_size();
|
let shard_size = e.shard_size();
|
||||||
let shard_file_size = e.shard_file_size(total_length);
|
let shard_file_size = e.shard_file_size(total_length);
|
||||||
|
|
||||||
@@ -42,7 +47,10 @@ impl ParallelReader {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ParallelReader {
|
impl<R> ParallelReader<R>
|
||||||
|
where
|
||||||
|
R: AsyncRead + Unpin + Send + Sync,
|
||||||
|
{
|
||||||
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
|
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
|
||||||
// if self.readers.len() != self.total_shards {
|
// if self.readers.len() != self.total_shards {
|
||||||
// return Err(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers"));
|
// return Err(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers"));
|
||||||
@@ -175,16 +183,17 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Erasure {
|
impl Erasure {
|
||||||
pub async fn decode<W>(
|
pub async fn decode<W, R>(
|
||||||
&self,
|
&self,
|
||||||
writer: &mut W,
|
writer: &mut W,
|
||||||
readers: Vec<Option<BitrotReader>>,
|
readers: Vec<Option<BitrotReader<R>>>,
|
||||||
offset: usize,
|
offset: usize,
|
||||||
length: usize,
|
length: usize,
|
||||||
total_length: usize,
|
total_length: usize,
|
||||||
) -> (usize, Option<std::io::Error>)
|
) -> (usize, Option<std::io::Error>)
|
||||||
where
|
where
|
||||||
W: tokio::io::AsyncWrite + Send + Sync + Unpin + 'static,
|
W: AsyncWrite + Send + Sync + Unpin,
|
||||||
|
R: AsyncRead + Unpin + Send + Sync,
|
||||||
{
|
{
|
||||||
if readers.len() != self.data_shards + self.parity_shards {
|
if readers.len() != self.data_shards + self.parity_shards {
|
||||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")));
|
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")));
|
||||||
|
|||||||
@@ -1,24 +1,22 @@
|
|||||||
use bytes::Bytes;
|
use super::BitrotWriterWrapper;
|
||||||
use rustfs_rio::BitrotWriter;
|
|
||||||
use rustfs_rio::Reader;
|
|
||||||
// use std::io::Cursor;
|
|
||||||
// use std::mem;
|
|
||||||
use super::Erasure;
|
use super::Erasure;
|
||||||
use crate::disk::error::Error;
|
use crate::disk::error::Error;
|
||||||
use crate::disk::error_reduce::count_errs;
|
use crate::disk::error_reduce::count_errs;
|
||||||
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs};
|
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs};
|
||||||
|
use bytes::Bytes;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::vec;
|
use std::vec;
|
||||||
|
use tokio::io::AsyncRead;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
|
|
||||||
pub(crate) struct MultiWriter<'a> {
|
pub(crate) struct MultiWriter<'a> {
|
||||||
writers: &'a mut [Option<BitrotWriter>],
|
writers: &'a mut [Option<BitrotWriterWrapper>],
|
||||||
write_quorum: usize,
|
write_quorum: usize,
|
||||||
errs: Vec<Option<Error>>,
|
errs: Vec<Option<Error>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> MultiWriter<'a> {
|
impl<'a> MultiWriter<'a> {
|
||||||
pub fn new(writers: &'a mut [Option<BitrotWriter>], write_quorum: usize) -> Self {
|
pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize) -> Self {
|
||||||
let length = writers.len();
|
let length = writers.len();
|
||||||
MultiWriter {
|
MultiWriter {
|
||||||
writers,
|
writers,
|
||||||
@@ -82,11 +80,11 @@ impl Erasure {
|
|||||||
pub async fn encode<R>(
|
pub async fn encode<R>(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
mut reader: R,
|
mut reader: R,
|
||||||
writers: &mut [Option<BitrotWriter>],
|
writers: &mut [Option<BitrotWriterWrapper>],
|
||||||
quorum: usize,
|
quorum: usize,
|
||||||
) -> std::io::Result<(R, usize)>
|
) -> std::io::Result<(R, usize)>
|
||||||
where
|
where
|
||||||
R: Reader + Send + Sync + Unpin + 'static,
|
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||||
{
|
{
|
||||||
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
|
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
|
||||||
|
|
||||||
|
|||||||
@@ -11,16 +11,16 @@
|
|||||||
//! - **Compatibility**: Works with any shard size
|
//! - **Compatibility**: Works with any shard size
|
||||||
//! - **Use case**: Default behavior, recommended for most production use cases
|
//! - **Use case**: Default behavior, recommended for most production use cases
|
||||||
//!
|
//!
|
||||||
//! ### Hybrid Mode (`reed-solomon-simd` feature)
|
//! ### SIMD Mode (`reed-solomon-simd` feature)
|
||||||
//! - **Performance**: Uses SIMD optimization when possible, falls back to erasure implementation for small shards
|
//! - **Performance**: Uses SIMD optimization for high-performance encoding/decoding
|
||||||
//! - **Compatibility**: Works with any shard size through intelligent fallback
|
//! - **Compatibility**: Works with any shard size through SIMD implementation
|
||||||
//! - **Reliability**: Best of both worlds - SIMD speed for large data, erasure stability for small data
|
//! - **Reliability**: High-performance SIMD implementation for large data processing
|
||||||
//! - **Use case**: Use when maximum performance is needed for large data processing
|
//! - **Use case**: Use when maximum performance is needed for large data processing
|
||||||
//!
|
//!
|
||||||
//! ## Feature Flags
|
//! ## Feature Flags
|
||||||
//!
|
//!
|
||||||
//! - Default: Use pure reed-solomon-erasure implementation (stable and reliable)
|
//! - Default: Use pure reed-solomon-erasure implementation (stable and reliable)
|
||||||
//! - `reed-solomon-simd`: Use hybrid mode (SIMD + erasure fallback for optimal performance)
|
//! - `reed-solomon-simd`: Use SIMD mode for optimal performance
|
||||||
//! - `reed-solomon-erasure`: Explicitly enable pure erasure mode (same as default)
|
//! - `reed-solomon-erasure`: Explicitly enable pure erasure mode (same as default)
|
||||||
//!
|
//!
|
||||||
//! ## Example
|
//! ## Example
|
||||||
@@ -38,25 +38,23 @@ use bytes::{Bytes, BytesMut};
|
|||||||
use reed_solomon_erasure::galois_8::ReedSolomon as ReedSolomonErasure;
|
use reed_solomon_erasure::galois_8::ReedSolomon as ReedSolomonErasure;
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
use reed_solomon_simd;
|
use reed_solomon_simd;
|
||||||
// use rustfs_rio::Reader;
|
|
||||||
use smallvec::SmallVec;
|
use smallvec::SmallVec;
|
||||||
use std::io;
|
use std::io;
|
||||||
|
use tokio::io::AsyncRead;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
/// Reed-Solomon encoder variants supporting different implementations.
|
/// Reed-Solomon encoder variants supporting different implementations.
|
||||||
#[allow(clippy::large_enum_variant)]
|
#[allow(clippy::large_enum_variant)]
|
||||||
pub enum ReedSolomonEncoder {
|
pub enum ReedSolomonEncoder {
|
||||||
/// Hybrid mode: SIMD with erasure fallback (when reed-solomon-simd feature is enabled)
|
/// SIMD mode: High-performance SIMD implementation (when reed-solomon-simd feature is enabled)
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
Hybrid {
|
SIMD {
|
||||||
data_shards: usize,
|
data_shards: usize,
|
||||||
parity_shards: usize,
|
parity_shards: usize,
|
||||||
// 使用RwLock确保线程安全,实现Send + Sync
|
// 使用RwLock确保线程安全,实现Send + Sync
|
||||||
encoder_cache: std::sync::RwLock<Option<reed_solomon_simd::ReedSolomonEncoder>>,
|
encoder_cache: std::sync::RwLock<Option<reed_solomon_simd::ReedSolomonEncoder>>,
|
||||||
decoder_cache: std::sync::RwLock<Option<reed_solomon_simd::ReedSolomonDecoder>>,
|
decoder_cache: std::sync::RwLock<Option<reed_solomon_simd::ReedSolomonDecoder>>,
|
||||||
// erasure fallback for small shards or SIMD failures
|
|
||||||
fallback_encoder: Box<ReedSolomonErasure>,
|
|
||||||
},
|
},
|
||||||
/// Pure erasure mode: default and when reed-solomon-erasure feature is specified
|
/// Pure erasure mode: default and when reed-solomon-erasure feature is specified
|
||||||
Erasure(Box<ReedSolomonErasure>),
|
Erasure(Box<ReedSolomonErasure>),
|
||||||
@@ -66,18 +64,16 @@ impl Clone for ReedSolomonEncoder {
|
|||||||
fn clone(&self) -> Self {
|
fn clone(&self) -> Self {
|
||||||
match self {
|
match self {
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
ReedSolomonEncoder::Hybrid {
|
ReedSolomonEncoder::SIMD {
|
||||||
data_shards,
|
data_shards,
|
||||||
parity_shards,
|
parity_shards,
|
||||||
fallback_encoder,
|
|
||||||
..
|
..
|
||||||
} => ReedSolomonEncoder::Hybrid {
|
} => ReedSolomonEncoder::SIMD {
|
||||||
data_shards: *data_shards,
|
data_shards: *data_shards,
|
||||||
parity_shards: *parity_shards,
|
parity_shards: *parity_shards,
|
||||||
// 为新实例创建空的缓存,不共享缓存
|
// 为新实例创建空的缓存,不共享缓存
|
||||||
encoder_cache: std::sync::RwLock::new(None),
|
encoder_cache: std::sync::RwLock::new(None),
|
||||||
decoder_cache: std::sync::RwLock::new(None),
|
decoder_cache: std::sync::RwLock::new(None),
|
||||||
fallback_encoder: fallback_encoder.clone(),
|
|
||||||
},
|
},
|
||||||
ReedSolomonEncoder::Erasure(encoder) => ReedSolomonEncoder::Erasure(encoder.clone()),
|
ReedSolomonEncoder::Erasure(encoder) => ReedSolomonEncoder::Erasure(encoder.clone()),
|
||||||
}
|
}
|
||||||
@@ -89,18 +85,12 @@ impl ReedSolomonEncoder {
|
|||||||
pub fn new(data_shards: usize, parity_shards: usize) -> io::Result<Self> {
|
pub fn new(data_shards: usize, parity_shards: usize) -> io::Result<Self> {
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
{
|
{
|
||||||
// Hybrid mode: SIMD + erasure fallback when reed-solomon-simd feature is enabled
|
// SIMD mode when reed-solomon-simd feature is enabled
|
||||||
let fallback_encoder = Box::new(
|
Ok(ReedSolomonEncoder::SIMD {
|
||||||
ReedSolomonErasure::new(data_shards, parity_shards)
|
|
||||||
.map_err(|e| io::Error::other(format!("Failed to create fallback erasure encoder: {:?}", e)))?,
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(ReedSolomonEncoder::Hybrid {
|
|
||||||
data_shards,
|
data_shards,
|
||||||
parity_shards,
|
parity_shards,
|
||||||
encoder_cache: std::sync::RwLock::new(None),
|
encoder_cache: std::sync::RwLock::new(None),
|
||||||
decoder_cache: std::sync::RwLock::new(None),
|
decoder_cache: std::sync::RwLock::new(None),
|
||||||
fallback_encoder,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,11 +107,10 @@ impl ReedSolomonEncoder {
|
|||||||
pub fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
|
pub fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
|
||||||
match self {
|
match self {
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
ReedSolomonEncoder::Hybrid {
|
ReedSolomonEncoder::SIMD {
|
||||||
data_shards,
|
data_shards,
|
||||||
parity_shards,
|
parity_shards,
|
||||||
encoder_cache,
|
encoder_cache,
|
||||||
fallback_encoder,
|
|
||||||
..
|
..
|
||||||
} => {
|
} => {
|
||||||
let mut shards_vec: Vec<&mut [u8]> = shards.into_vec();
|
let mut shards_vec: Vec<&mut [u8]> = shards.into_vec();
|
||||||
@@ -129,30 +118,14 @@ impl ReedSolomonEncoder {
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let shard_len = shards_vec[0].len();
|
// 使用 SIMD 进行编码
|
||||||
|
|
||||||
// SIMD 性能最佳的最小 shard 大小 (通常 512-1024 字节)
|
|
||||||
const SIMD_MIN_SHARD_SIZE: usize = 512;
|
|
||||||
|
|
||||||
// 如果 shard 太小,直接使用 fallback encoder
|
|
||||||
if shard_len < SIMD_MIN_SHARD_SIZE {
|
|
||||||
let fallback_shards: SmallVec<[&mut [u8]; 16]> = SmallVec::from_vec(shards_vec);
|
|
||||||
return fallback_encoder
|
|
||||||
.encode(fallback_shards)
|
|
||||||
.map_err(|e| io::Error::other(format!("Fallback erasure encode error: {:?}", e)));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 尝试使用 SIMD,如果失败则回退到 fallback
|
|
||||||
let simd_result = self.encode_with_simd(*data_shards, *parity_shards, encoder_cache, &mut shards_vec);
|
let simd_result = self.encode_with_simd(*data_shards, *parity_shards, encoder_cache, &mut shards_vec);
|
||||||
|
|
||||||
match simd_result {
|
match simd_result {
|
||||||
Ok(()) => Ok(()),
|
Ok(()) => Ok(()),
|
||||||
Err(simd_error) => {
|
Err(simd_error) => {
|
||||||
warn!("SIMD encoding failed: {}, using fallback", simd_error);
|
warn!("SIMD encoding failed: {}", simd_error);
|
||||||
let fallback_shards: SmallVec<[&mut [u8]; 16]> = SmallVec::from_vec(shards_vec);
|
Err(simd_error)
|
||||||
fallback_encoder
|
|
||||||
.encode(fallback_shards)
|
|
||||||
.map_err(|e| io::Error::other(format!("Fallback erasure encode error: {:?}", e)))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -231,39 +204,20 @@ impl ReedSolomonEncoder {
|
|||||||
pub fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
|
pub fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
|
||||||
match self {
|
match self {
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
ReedSolomonEncoder::Hybrid {
|
ReedSolomonEncoder::SIMD {
|
||||||
data_shards,
|
data_shards,
|
||||||
parity_shards,
|
parity_shards,
|
||||||
decoder_cache,
|
decoder_cache,
|
||||||
fallback_encoder,
|
|
||||||
..
|
..
|
||||||
} => {
|
} => {
|
||||||
// Find a valid shard to determine length
|
// 使用 SIMD 进行重构
|
||||||
let shard_len = shards
|
|
||||||
.iter()
|
|
||||||
.find_map(|s| s.as_ref().map(|v| v.len()))
|
|
||||||
.ok_or_else(|| io::Error::other("No valid shards found for reconstruction"))?;
|
|
||||||
|
|
||||||
// SIMD 性能最佳的最小 shard 大小
|
|
||||||
const SIMD_MIN_SHARD_SIZE: usize = 512;
|
|
||||||
|
|
||||||
// 如果 shard 太小,直接使用 fallback encoder
|
|
||||||
if shard_len < SIMD_MIN_SHARD_SIZE {
|
|
||||||
return fallback_encoder
|
|
||||||
.reconstruct(shards)
|
|
||||||
.map_err(|e| io::Error::other(format!("Fallback erasure reconstruct error: {:?}", e)));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 尝试使用 SIMD,如果失败则回退到 fallback
|
|
||||||
let simd_result = self.reconstruct_with_simd(*data_shards, *parity_shards, decoder_cache, shards);
|
let simd_result = self.reconstruct_with_simd(*data_shards, *parity_shards, decoder_cache, shards);
|
||||||
|
|
||||||
match simd_result {
|
match simd_result {
|
||||||
Ok(()) => Ok(()),
|
Ok(()) => Ok(()),
|
||||||
Err(simd_error) => {
|
Err(simd_error) => {
|
||||||
warn!("SIMD reconstruction failed: {}, using fallback", simd_error);
|
warn!("SIMD reconstruction failed: {}", simd_error);
|
||||||
fallback_encoder
|
Err(simd_error)
|
||||||
.reconstruct(shards)
|
|
||||||
.map_err(|e| io::Error::other(format!("Fallback erasure reconstruct error: {:?}", e)))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -402,6 +356,10 @@ impl Clone for Erasure {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize {
|
||||||
|
(block_size.div_ceil(data_shards) + 1) & !1
|
||||||
|
}
|
||||||
|
|
||||||
impl Erasure {
|
impl Erasure {
|
||||||
/// Create a new Erasure instance.
|
/// Create a new Erasure instance.
|
||||||
///
|
///
|
||||||
@@ -439,7 +397,7 @@ impl Erasure {
|
|||||||
// let total_size = shard_size * self.total_shard_count();
|
// let total_size = shard_size * self.total_shard_count();
|
||||||
|
|
||||||
// 数据切片数量
|
// 数据切片数量
|
||||||
let per_shard_size = data.len().div_ceil(self.data_shards);
|
let per_shard_size = calc_shard_size(data.len(), self.data_shards);
|
||||||
// 总需求大小
|
// 总需求大小
|
||||||
let need_total_size = per_shard_size * self.total_shard_count();
|
let need_total_size = per_shard_size * self.total_shard_count();
|
||||||
|
|
||||||
@@ -507,7 +465,7 @@ impl Erasure {
|
|||||||
|
|
||||||
/// Calculate the size of each shard.
|
/// Calculate the size of each shard.
|
||||||
pub fn shard_size(&self) -> usize {
|
pub fn shard_size(&self) -> usize {
|
||||||
self.block_size.div_ceil(self.data_shards)
|
calc_shard_size(self.block_size, self.data_shards)
|
||||||
}
|
}
|
||||||
/// Calculate the total erasure file size for a given original size.
|
/// Calculate the total erasure file size for a given original size.
|
||||||
// Returns the final erasure size from the original size
|
// Returns the final erasure size from the original size
|
||||||
@@ -518,7 +476,7 @@ impl Erasure {
|
|||||||
|
|
||||||
let num_shards = total_length / self.block_size;
|
let num_shards = total_length / self.block_size;
|
||||||
let last_block_size = total_length % self.block_size;
|
let last_block_size = total_length % self.block_size;
|
||||||
let last_shard_size = last_block_size.div_ceil(self.data_shards);
|
let last_shard_size = calc_shard_size(last_block_size, self.data_shards);
|
||||||
num_shards * self.shard_size() + last_shard_size
|
num_shards * self.shard_size() + last_shard_size
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -536,22 +494,29 @@ impl Erasure {
|
|||||||
till_offset
|
till_offset
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Encode all data from a rustfs_rio::Reader in blocks, calling an async callback for each encoded block.
|
/// Encode all data from a reader in blocks, calling an async callback for each encoded block.
|
||||||
/// This method is async and returns the reader and total bytes read after all blocks are processed.
|
/// This method is async and returns the total bytes read after all blocks are processed.
|
||||||
///
|
///
|
||||||
/// # Arguments
|
/// # Arguments
|
||||||
/// * `reader` - A rustfs_rio::Reader to read data from.
|
/// * `reader` - An async reader implementing AsyncRead + Send + Sync + Unpin
|
||||||
/// * `mut on_block` - Async callback: FnMut(Result<Vec<Bytes>, std::io::Error>) -> Future<Output=Result<(), E>> + Send
|
/// * `mut on_block` - Async callback that receives encoded blocks and returns a Result
|
||||||
|
/// * `F` - Callback type: FnMut(Result<Vec<Bytes>, std::io::Error>) -> Future<Output=Result<(), E>> + Send
|
||||||
|
/// * `Fut` - Future type returned by the callback
|
||||||
|
/// * `E` - Error type returned by the callback
|
||||||
|
/// * `R` - Reader type implementing AsyncRead + Send + Sync + Unpin
|
||||||
///
|
///
|
||||||
/// # Returns
|
/// # Returns
|
||||||
/// Result<(reader, total_bytes_read), E> after all data has been processed or on callback error.
|
/// Result<usize, E> containing total bytes read, or error from callback
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns error if reading from reader fails or if callback returns error
|
||||||
pub async fn encode_stream_callback_async<F, Fut, E, R>(
|
pub async fn encode_stream_callback_async<F, Fut, E, R>(
|
||||||
self: std::sync::Arc<Self>,
|
self: std::sync::Arc<Self>,
|
||||||
reader: &mut R,
|
reader: &mut R,
|
||||||
mut on_block: F,
|
mut on_block: F,
|
||||||
) -> Result<usize, E>
|
) -> Result<usize, E>
|
||||||
where
|
where
|
||||||
R: rustfs_rio::Reader + Send + Sync + Unpin,
|
R: AsyncRead + Send + Sync + Unpin,
|
||||||
F: FnMut(std::io::Result<Vec<Bytes>>) -> Fut + Send,
|
F: FnMut(std::io::Result<Vec<Bytes>>) -> Fut + Send,
|
||||||
Fut: std::future::Future<Output = Result<(), E>> + Send,
|
Fut: std::future::Future<Output = Result<(), E>> + Send,
|
||||||
{
|
{
|
||||||
@@ -582,6 +547,7 @@ impl Erasure {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -603,9 +569,9 @@ mod tests {
|
|||||||
// Case 5: total_length > block_size, aligned
|
// Case 5: total_length > block_size, aligned
|
||||||
assert_eq!(erasure.shard_file_size(16), 4); // 16/8=2, last=0, 2*2+0=4
|
assert_eq!(erasure.shard_file_size(16), 4); // 16/8=2, last=0, 2*2+0=4
|
||||||
|
|
||||||
assert_eq!(erasure.shard_file_size(1248739), 312185); // 1248739/8=156092, last=3, 3 div_ceil 4=1, 156092*2+1=312185
|
assert_eq!(erasure.shard_file_size(1248739), 312186); // 1248739/8=156092, last=3, 3 div_ceil 4=1, 156092*2+1=312185
|
||||||
|
|
||||||
assert_eq!(erasure.shard_file_size(43), 11); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11
|
assert_eq!(erasure.shard_file_size(43), 12); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -617,7 +583,7 @@ mod tests {
|
|||||||
#[cfg(not(feature = "reed-solomon-simd"))]
|
#[cfg(not(feature = "reed-solomon-simd"))]
|
||||||
let block_size = 8; // Pure erasure mode (default)
|
let block_size = 8; // Pure erasure mode (default)
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
let block_size = 1024; // Hybrid mode - SIMD with fallback
|
let block_size = 1024; // SIMD mode - SIMD with fallback
|
||||||
|
|
||||||
let erasure = Erasure::new(data_shards, parity_shards, block_size);
|
let erasure = Erasure::new(data_shards, parity_shards, block_size);
|
||||||
|
|
||||||
@@ -625,7 +591,7 @@ mod tests {
|
|||||||
#[cfg(not(feature = "reed-solomon-simd"))]
|
#[cfg(not(feature = "reed-solomon-simd"))]
|
||||||
let test_data = b"hello world".to_vec(); // Small data for erasure (default)
|
let test_data = b"hello world".to_vec(); // Small data for erasure (default)
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
let test_data = b"Hybrid mode test data for encoding and decoding roundtrip verification with sufficient length to ensure shard size requirements are met for proper SIMD optimization.".repeat(20); // ~3KB for hybrid
|
let test_data = b"SIMD mode test data for encoding and decoding roundtrip verification with sufficient length to ensure shard size requirements are met for proper SIMD optimization.".repeat(20); // ~3KB for SIMD
|
||||||
|
|
||||||
let data = &test_data;
|
let data = &test_data;
|
||||||
let encoded_shards = erasure.encode_data(data).unwrap();
|
let encoded_shards = erasure.encode_data(data).unwrap();
|
||||||
@@ -655,7 +621,7 @@ mod tests {
|
|||||||
|
|
||||||
// Use different block sizes based on feature
|
// Use different block sizes based on feature
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
let block_size = 512 * 3; // Hybrid mode - SIMD with fallback
|
let block_size = 512 * 3; // SIMD mode
|
||||||
#[cfg(not(feature = "reed-solomon-simd"))]
|
#[cfg(not(feature = "reed-solomon-simd"))]
|
||||||
let block_size = 8192; // Pure erasure mode (default)
|
let block_size = 8192; // Pure erasure mode (default)
|
||||||
|
|
||||||
@@ -700,7 +666,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_shard_size_and_file_size() {
|
fn test_shard_size_and_file_size() {
|
||||||
let erasure = Erasure::new(4, 2, 8);
|
let erasure = Erasure::new(4, 2, 8);
|
||||||
assert_eq!(erasure.shard_file_size(33), 9);
|
assert_eq!(erasure.shard_file_size(33), 10);
|
||||||
assert_eq!(erasure.shard_file_size(0), 0);
|
assert_eq!(erasure.shard_file_size(0), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -722,7 +688,7 @@ mod tests {
|
|||||||
|
|
||||||
// Use different block sizes based on feature
|
// Use different block sizes based on feature
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
let block_size = 1024; // Hybrid mode
|
let block_size = 1024; // SIMD mode
|
||||||
#[cfg(not(feature = "reed-solomon-simd"))]
|
#[cfg(not(feature = "reed-solomon-simd"))]
|
||||||
let block_size = 8; // Pure erasure mode (default)
|
let block_size = 8; // Pure erasure mode (default)
|
||||||
|
|
||||||
@@ -732,12 +698,12 @@ mod tests {
|
|||||||
let data =
|
let data =
|
||||||
b"Async error test data with sufficient length to meet requirements for proper testing and validation.".repeat(20); // ~2KB
|
b"Async error test data with sufficient length to meet requirements for proper testing and validation.".repeat(20); // ~2KB
|
||||||
|
|
||||||
let mut rio_reader = Cursor::new(data);
|
let mut reader = Cursor::new(data);
|
||||||
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
|
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
|
||||||
let erasure_clone = erasure.clone();
|
let erasure_clone = erasure.clone();
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
erasure_clone
|
erasure_clone
|
||||||
.encode_stream_callback_async::<_, _, (), _>(&mut rio_reader, move |res| {
|
.encode_stream_callback_async::<_, _, (), _>(&mut reader, move |res| {
|
||||||
let tx = tx.clone();
|
let tx = tx.clone();
|
||||||
async move {
|
async move {
|
||||||
let shards = res.unwrap();
|
let shards = res.unwrap();
|
||||||
@@ -765,7 +731,7 @@ mod tests {
|
|||||||
|
|
||||||
// Use different block sizes based on feature
|
// Use different block sizes based on feature
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
let block_size = 1024; // Hybrid mode
|
let block_size = 1024; // SIMD mode
|
||||||
#[cfg(not(feature = "reed-solomon-simd"))]
|
#[cfg(not(feature = "reed-solomon-simd"))]
|
||||||
let block_size = 8; // Pure erasure mode (default)
|
let block_size = 8; // Pure erasure mode (default)
|
||||||
|
|
||||||
@@ -779,12 +745,12 @@ mod tests {
|
|||||||
// let data = b"callback".to_vec(); // 8 bytes to fit exactly in one 8-byte block
|
// let data = b"callback".to_vec(); // 8 bytes to fit exactly in one 8-byte block
|
||||||
|
|
||||||
let data_clone = data.clone(); // Clone for later comparison
|
let data_clone = data.clone(); // Clone for later comparison
|
||||||
let mut rio_reader = Cursor::new(data);
|
let mut reader = Cursor::new(data);
|
||||||
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
|
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
|
||||||
let erasure_clone = erasure.clone();
|
let erasure_clone = erasure.clone();
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
erasure_clone
|
erasure_clone
|
||||||
.encode_stream_callback_async::<_, _, (), _>(&mut rio_reader, move |res| {
|
.encode_stream_callback_async::<_, _, (), _>(&mut reader, move |res| {
|
||||||
let tx = tx.clone();
|
let tx = tx.clone();
|
||||||
async move {
|
async move {
|
||||||
let shards = res.unwrap();
|
let shards = res.unwrap();
|
||||||
@@ -816,20 +782,20 @@ mod tests {
|
|||||||
assert_eq!(&recovered, &data_clone);
|
assert_eq!(&recovered, &data_clone);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tests specifically for hybrid mode (SIMD + erasure fallback)
|
// Tests specifically for SIMD mode
|
||||||
#[cfg(feature = "reed-solomon-simd")]
|
#[cfg(feature = "reed-solomon-simd")]
|
||||||
mod hybrid_tests {
|
mod simd_tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_hybrid_encode_decode_roundtrip() {
|
fn test_simd_encode_decode_roundtrip() {
|
||||||
let data_shards = 4;
|
let data_shards = 4;
|
||||||
let parity_shards = 2;
|
let parity_shards = 2;
|
||||||
let block_size = 1024; // Use larger block size for hybrid mode
|
let block_size = 1024; // Use larger block size for SIMD mode
|
||||||
let erasure = Erasure::new(data_shards, parity_shards, block_size);
|
let erasure = Erasure::new(data_shards, parity_shards, block_size);
|
||||||
|
|
||||||
// Use data that will create shards >= 512 bytes for SIMD optimization
|
// Use data that will create shards >= 512 bytes for SIMD optimization
|
||||||
let test_data = b"Hybrid test data for encoding and decoding roundtrip verification with sufficient length to ensure shard size requirements are met for proper SIMD optimization and validation.";
|
let test_data = b"SIMD mode test data for encoding and decoding roundtrip verification with sufficient length to ensure shard size requirements are met for proper SIMD optimization and validation.";
|
||||||
let data = test_data.repeat(25); // Create much larger data: ~5KB total, ~1.25KB per shard
|
let data = test_data.repeat(25); // Create much larger data: ~5KB total, ~1.25KB per shard
|
||||||
|
|
||||||
let encoded_shards = erasure.encode_data(&data).unwrap();
|
let encoded_shards = erasure.encode_data(&data).unwrap();
|
||||||
@@ -854,10 +820,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_hybrid_all_zero_data() {
|
fn test_simd_all_zero_data() {
|
||||||
let data_shards = 4;
|
let data_shards = 4;
|
||||||
let parity_shards = 2;
|
let parity_shards = 2;
|
||||||
let block_size = 1024; // Use larger block size for hybrid mode
|
let block_size = 1024; // Use larger block size for SIMD mode
|
||||||
let erasure = Erasure::new(data_shards, parity_shards, block_size);
|
let erasure = Erasure::new(data_shards, parity_shards, block_size);
|
||||||
|
|
||||||
// Create all-zero data that ensures adequate shard size for SIMD optimization
|
// Create all-zero data that ensures adequate shard size for SIMD optimization
|
||||||
@@ -997,39 +963,35 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_simd_smart_fallback() {
|
fn test_simd_small_data_handling() {
|
||||||
let data_shards = 4;
|
let data_shards = 4;
|
||||||
let parity_shards = 2;
|
let parity_shards = 2;
|
||||||
let block_size = 32; // 很小的block_size,会导致小shard
|
let block_size = 32; // Small block size for testing edge cases
|
||||||
let erasure = Erasure::new(data_shards, parity_shards, block_size);
|
let erasure = Erasure::new(data_shards, parity_shards, block_size);
|
||||||
|
|
||||||
// 使用小数据,每个shard只有8字节,远小于512字节SIMD最小要求
|
// Use small data to test SIMD handling of small shards
|
||||||
let small_data = b"tiny!123".to_vec(); // 8字节数据
|
let small_data = b"tiny!123".to_vec(); // 8 bytes data
|
||||||
|
|
||||||
// 应该能够成功编码(通过fallback)
|
// Test encoding with small data
|
||||||
let result = erasure.encode_data(&small_data);
|
let result = erasure.encode_data(&small_data);
|
||||||
match result {
|
match result {
|
||||||
Ok(shards) => {
|
Ok(shards) => {
|
||||||
println!(
|
println!("✅ SIMD encoding succeeded: {} bytes into {} shards", small_data.len(), shards.len());
|
||||||
"✅ Smart fallback worked: encoded {} bytes into {} shards",
|
|
||||||
small_data.len(),
|
|
||||||
shards.len()
|
|
||||||
);
|
|
||||||
assert_eq!(shards.len(), data_shards + parity_shards);
|
assert_eq!(shards.len(), data_shards + parity_shards);
|
||||||
|
|
||||||
// 测试解码
|
// Test decoding
|
||||||
let mut shards_opt: Vec<Option<Vec<u8>>> = shards.iter().map(|shard| Some(shard.to_vec())).collect();
|
let mut shards_opt: Vec<Option<Vec<u8>>> = shards.iter().map(|shard| Some(shard.to_vec())).collect();
|
||||||
|
|
||||||
// 丢失一些shard来测试恢复
|
// Lose some shards to test recovery
|
||||||
shards_opt[1] = None; // 丢失一个数据shard
|
shards_opt[1] = None; // Lose one data shard
|
||||||
shards_opt[4] = None; // 丢失一个奇偶shard
|
shards_opt[4] = None; // Lose one parity shard
|
||||||
|
|
||||||
let decode_result = erasure.decode_data(&mut shards_opt);
|
let decode_result = erasure.decode_data(&mut shards_opt);
|
||||||
match decode_result {
|
match decode_result {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
println!("✅ Smart fallback decode worked");
|
println!("✅ SIMD decode worked");
|
||||||
|
|
||||||
// 验证恢复的数据
|
// Verify recovered data
|
||||||
let mut recovered = Vec::new();
|
let mut recovered = Vec::new();
|
||||||
for shard in shards_opt.iter().take(data_shards) {
|
for shard in shards_opt.iter().take(data_shards) {
|
||||||
recovered.extend_from_slice(shard.as_ref().unwrap());
|
recovered.extend_from_slice(shard.as_ref().unwrap());
|
||||||
@@ -1038,17 +1000,17 @@ mod tests {
|
|||||||
println!("recovered: {:?}", recovered);
|
println!("recovered: {:?}", recovered);
|
||||||
println!("small_data: {:?}", small_data);
|
println!("small_data: {:?}", small_data);
|
||||||
assert_eq!(&recovered, &small_data);
|
assert_eq!(&recovered, &small_data);
|
||||||
println!("✅ Data recovery successful with smart fallback");
|
println!("✅ Data recovery successful with SIMD");
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
println!("❌ Smart fallback decode failed: {}", e);
|
println!("❌ SIMD decode failed: {}", e);
|
||||||
// 对于很小的数据,如果decode失败也是可以接受的
|
// For very small data, decode failure might be acceptable
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
println!("❌ Smart fallback encode failed: {}", e);
|
println!("❌ SIMD encode failed: {}", e);
|
||||||
// 如果连fallback都失败了,说明数据太小或配置有问题
|
// For very small data or configuration issues, encoding might fail
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1143,14 +1105,14 @@ mod tests {
|
|||||||
let test_data = b"SIMD stream processing test with sufficient data length for multiple blocks and proper SIMD optimization verification!";
|
let test_data = b"SIMD stream processing test with sufficient data length for multiple blocks and proper SIMD optimization verification!";
|
||||||
let data = test_data.repeat(5); // Create owned Vec<u8>
|
let data = test_data.repeat(5); // Create owned Vec<u8>
|
||||||
let data_clone = data.clone(); // Clone for later comparison
|
let data_clone = data.clone(); // Clone for later comparison
|
||||||
let mut rio_reader = Cursor::new(data);
|
let mut reader = Cursor::new(data);
|
||||||
|
|
||||||
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(16);
|
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(16);
|
||||||
let erasure_clone = erasure.clone();
|
let erasure_clone = erasure.clone();
|
||||||
|
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
erasure_clone
|
erasure_clone
|
||||||
.encode_stream_callback_async::<_, _, (), _>(&mut rio_reader, move |res| {
|
.encode_stream_callback_async::<_, _, (), _>(&mut reader, move |res| {
|
||||||
let tx = tx.clone();
|
let tx = tx.clone();
|
||||||
async move {
|
async move {
|
||||||
let shards = res.unwrap();
|
let shards = res.unwrap();
|
||||||
|
|||||||
@@ -1,19 +1,23 @@
|
|||||||
|
use super::BitrotReader;
|
||||||
|
use super::BitrotWriterWrapper;
|
||||||
use super::decode::ParallelReader;
|
use super::decode::ParallelReader;
|
||||||
use crate::disk::error::{Error, Result};
|
use crate::disk::error::{Error, Result};
|
||||||
use crate::erasure_coding::encode::MultiWriter;
|
use crate::erasure_coding::encode::MultiWriter;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use rustfs_rio::BitrotReader;
|
use tokio::io::AsyncRead;
|
||||||
use rustfs_rio::BitrotWriter;
|
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
impl super::Erasure {
|
impl super::Erasure {
|
||||||
pub async fn heal(
|
pub async fn heal<R>(
|
||||||
&self,
|
&self,
|
||||||
writers: &mut [Option<BitrotWriter>],
|
writers: &mut [Option<BitrotWriterWrapper>],
|
||||||
readers: Vec<Option<BitrotReader>>,
|
readers: Vec<Option<BitrotReader<R>>>,
|
||||||
total_length: usize,
|
total_length: usize,
|
||||||
_prefer: &[bool],
|
_prefer: &[bool],
|
||||||
) -> Result<()> {
|
) -> Result<()>
|
||||||
|
where
|
||||||
|
R: AsyncRead + Unpin + Send + Sync,
|
||||||
|
{
|
||||||
info!(
|
info!(
|
||||||
"Erasure heal, writers len: {}, readers len: {}, total_length: {}",
|
"Erasure heal, writers len: {}, readers len: {}, total_length: {}",
|
||||||
writers.len(),
|
writers.len(),
|
||||||
|
|||||||
@@ -3,4 +3,7 @@ pub mod encode;
|
|||||||
pub mod erasure;
|
pub mod erasure;
|
||||||
pub mod heal;
|
pub mod heal;
|
||||||
|
|
||||||
|
mod bitrot;
|
||||||
|
pub use bitrot::*;
|
||||||
|
|
||||||
pub use erasure::{Erasure, ReedSolomonEncoder};
|
pub use erasure::{Erasure, ReedSolomonEncoder};
|
||||||
|
|||||||
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
pub mod admin_server_info;
|
pub mod admin_server_info;
|
||||||
// pub mod bitrot;
|
pub mod bitrot;
|
||||||
pub mod bucket;
|
pub mod bucket;
|
||||||
pub mod cache_value;
|
pub mod cache_value;
|
||||||
mod chunk_stream;
|
mod chunk_stream;
|
||||||
|
|||||||
+168
-182
@@ -1,9 +1,11 @@
|
|||||||
|
use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
|
||||||
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
|
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
|
||||||
use crate::disk::{
|
use crate::disk::{
|
||||||
self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS,
|
self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS,
|
||||||
conv_part_err_to_int, has_part_err,
|
conv_part_err_to_int, has_part_err,
|
||||||
};
|
};
|
||||||
use crate::erasure_coding;
|
use crate::erasure_coding;
|
||||||
|
use crate::erasure_coding::bitrot_verify;
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
use crate::global::GLOBAL_MRFState;
|
use crate::global::GLOBAL_MRFState;
|
||||||
use crate::heal::data_usage_cache::DataUsageCache;
|
use crate::heal::data_usage_cache::DataUsageCache;
|
||||||
@@ -64,7 +66,7 @@ use rustfs_filemeta::{
|
|||||||
FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo,
|
FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo,
|
||||||
RawFileInfo, file_info_from_raw, merge_file_meta_versions,
|
RawFileInfo, file_info_from_raw, merge_file_meta_versions,
|
||||||
};
|
};
|
||||||
use rustfs_rio::{BitrotReader, BitrotWriter, EtagResolvable, HashReader, Writer, bitrot_verify};
|
use rustfs_rio::{EtagResolvable, HashReader};
|
||||||
use rustfs_utils::HashAlgorithm;
|
use rustfs_utils::HashAlgorithm;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::hash::Hash;
|
use std::hash::Hash;
|
||||||
@@ -1865,51 +1867,31 @@ impl SetDisks {
|
|||||||
let mut readers = Vec::with_capacity(disks.len());
|
let mut readers = Vec::with_capacity(disks.len());
|
||||||
let mut errors = Vec::with_capacity(disks.len());
|
let mut errors = Vec::with_capacity(disks.len());
|
||||||
for (idx, disk_op) in disks.iter().enumerate() {
|
for (idx, disk_op) in disks.iter().enumerate() {
|
||||||
if let Some(inline_data) = files[idx].data.clone() {
|
match create_bitrot_reader(
|
||||||
let rd = Cursor::new(inline_data);
|
files[idx].data.as_deref(),
|
||||||
let reader = BitrotReader::new(Box::new(rd), erasure.shard_size(), HashAlgorithm::HighwayHash256);
|
disk_op.as_ref(),
|
||||||
readers.push(Some(reader));
|
bucket,
|
||||||
errors.push(None);
|
&format!("{}/{}/part.{}", object, files[idx].data_dir.unwrap_or_default(), part_number),
|
||||||
} else if let Some(disk) = disk_op {
|
part_offset,
|
||||||
// Calculate ceiling division of till_offset by shard_size
|
till_offset,
|
||||||
let till_offset =
|
erasure.shard_size(),
|
||||||
till_offset.div_ceil(erasure.shard_size()) * HashAlgorithm::HighwayHash256.size() + till_offset;
|
HashAlgorithm::HighwayHash256,
|
||||||
|
)
|
||||||
let rd = disk
|
.await
|
||||||
.read_file_stream(
|
{
|
||||||
bucket,
|
Ok(Some(reader)) => {
|
||||||
&format!("{}/{}/part.{}", object, files[idx].data_dir.unwrap_or(Uuid::nil()), part_number),
|
readers.push(Some(reader));
|
||||||
part_offset,
|
errors.push(None);
|
||||||
till_offset,
|
}
|
||||||
)
|
Ok(None) => {
|
||||||
.await?;
|
readers.push(None);
|
||||||
let reader = BitrotReader::new(
|
errors.push(Some(DiskError::DiskNotFound));
|
||||||
Box::new(rustfs_rio::WarpReader::new(rd)),
|
}
|
||||||
erasure.shard_size(),
|
Err(e) => {
|
||||||
HashAlgorithm::HighwayHash256,
|
readers.push(None);
|
||||||
);
|
errors.push(Some(e));
|
||||||
readers.push(Some(reader));
|
}
|
||||||
errors.push(None);
|
|
||||||
} else {
|
|
||||||
errors.push(Some(DiskError::DiskNotFound));
|
|
||||||
readers.push(None);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// if let Some(disk) = disk_op {
|
|
||||||
// let checksum_info = files[idx].erasure.get_checksum_info(part_number);
|
|
||||||
// let reader = new_bitrot_filereader(
|
|
||||||
// disk.clone(),
|
|
||||||
// files[idx].data.clone(),
|
|
||||||
// bucket.to_owned(),
|
|
||||||
// format!("{}/{}/part.{}", object, files[idx].data_dir.unwrap_or(Uuid::nil()), part_number),
|
|
||||||
// till_offset,
|
|
||||||
// checksum_info.algorithm,
|
|
||||||
// erasure.shard_size(erasure.block_size),
|
|
||||||
// );
|
|
||||||
// readers.push(Some(reader));
|
|
||||||
// } else {
|
|
||||||
// readers.push(None)
|
|
||||||
// }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
|
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
|
||||||
@@ -2478,59 +2460,31 @@ impl SetDisks {
|
|||||||
let mut prefer = vec![false; latest_disks.len()];
|
let mut prefer = vec![false; latest_disks.len()];
|
||||||
for (index, disk) in latest_disks.iter().enumerate() {
|
for (index, disk) in latest_disks.iter().enumerate() {
|
||||||
if let (Some(disk), Some(metadata)) = (disk, ©_parts_metadata[index]) {
|
if let (Some(disk), Some(metadata)) = (disk, ©_parts_metadata[index]) {
|
||||||
// let filereader = {
|
match create_bitrot_reader(
|
||||||
// if let Some(ref data) = metadata.data {
|
metadata.data.as_deref(),
|
||||||
// Box::new(BufferReader::new(data.clone()))
|
Some(disk),
|
||||||
// } else {
|
bucket,
|
||||||
// let disk = disk.clone();
|
&format!("{}/{}/part.{}", object, src_data_dir, part.number),
|
||||||
// let part_path = format!("{}/{}/part.{}", object, src_data_dir, part.number);
|
0,
|
||||||
|
till_offset,
|
||||||
// disk.read_file(bucket, &part_path).await?
|
erasure.shard_size(),
|
||||||
// }
|
checksum_algo.clone(),
|
||||||
// };
|
)
|
||||||
// let reader = new_bitrot_filereader(
|
.await
|
||||||
// disk.clone(),
|
{
|
||||||
// metadata.data.clone(),
|
Ok(Some(reader)) => {
|
||||||
// bucket.to_owned(),
|
readers.push(Some(reader));
|
||||||
// format!("{}/{}/part.{}", object, src_data_dir, part.number),
|
}
|
||||||
// till_offset,
|
Ok(None) => {
|
||||||
// checksum_algo.clone(),
|
error!("heal_object disk not available");
|
||||||
// erasure.shard_size(erasure.block_size),
|
readers.push(None);
|
||||||
// );
|
continue;
|
||||||
|
}
|
||||||
if let Some(ref data) = metadata.data {
|
Err(e) => {
|
||||||
let rd = Cursor::new(data.clone());
|
error!("heal_object read_file err: {:?}", e);
|
||||||
let reader =
|
readers.push(None);
|
||||||
BitrotReader::new(Box::new(rd), erasure.shard_size(), checksum_algo.clone());
|
continue;
|
||||||
readers.push(Some(reader));
|
}
|
||||||
// errors.push(None);
|
|
||||||
} else {
|
|
||||||
let length =
|
|
||||||
till_offset.div_ceil(erasure.shard_size()) * checksum_algo.size() + till_offset;
|
|
||||||
let rd = match disk
|
|
||||||
.read_file_stream(
|
|
||||||
bucket,
|
|
||||||
&format!("{}/{}/part.{}", object, src_data_dir, part.number),
|
|
||||||
0,
|
|
||||||
length,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(rd) => rd,
|
|
||||||
Err(e) => {
|
|
||||||
// errors.push(Some(e.into()));
|
|
||||||
error!("heal_object read_file err: {:?}", e);
|
|
||||||
writers.push(None);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let reader = BitrotReader::new(
|
|
||||||
Box::new(rustfs_rio::WarpReader::new(rd)),
|
|
||||||
erasure.shard_size(),
|
|
||||||
HashAlgorithm::HighwayHash256,
|
|
||||||
);
|
|
||||||
readers.push(Some(reader));
|
|
||||||
// errors.push(None);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
prefer[index] = disk.host_name().is_empty();
|
prefer[index] = disk.host_name().is_empty();
|
||||||
@@ -2549,55 +2503,67 @@ impl SetDisks {
|
|||||||
};
|
};
|
||||||
|
|
||||||
for disk in out_dated_disks.iter() {
|
for disk in out_dated_disks.iter() {
|
||||||
if let Some(disk) = disk {
|
let writer = create_bitrot_writer(
|
||||||
// let filewriter = {
|
is_inline_buffer,
|
||||||
// if is_inline_buffer {
|
disk.as_ref(),
|
||||||
// Box::new(Cursor::new(Vec::new()))
|
RUSTFS_META_TMP_BUCKET,
|
||||||
// } else {
|
&format!("{}/{}/part.{}", tmp_id, dst_data_dir, part.number),
|
||||||
// let disk = disk.clone();
|
erasure.shard_file_size(part.size),
|
||||||
// let part_path = format!("{}/{}/part.{}", tmp_id, dst_data_dir, part.number);
|
erasure.shard_size(),
|
||||||
// disk.create_file("", RUSTFS_META_TMP_BUCKET, &part_path, 0).await?
|
HashAlgorithm::HighwayHash256,
|
||||||
// }
|
)
|
||||||
// };
|
.await?;
|
||||||
|
writers.push(Some(writer));
|
||||||
|
|
||||||
if is_inline_buffer {
|
// if let Some(disk) = disk {
|
||||||
let writer = BitrotWriter::new(
|
// // let filewriter = {
|
||||||
Writer::from_cursor(Cursor::new(Vec::new())),
|
// // if is_inline_buffer {
|
||||||
erasure.shard_size(),
|
// // Box::new(Cursor::new(Vec::new()))
|
||||||
HashAlgorithm::HighwayHash256,
|
// // } else {
|
||||||
);
|
// // let disk = disk.clone();
|
||||||
writers.push(Some(writer));
|
// // let part_path = format!("{}/{}/part.{}", tmp_id, dst_data_dir, part.number);
|
||||||
} else {
|
// // disk.create_file("", RUSTFS_META_TMP_BUCKET, &part_path, 0).await?
|
||||||
let f = disk
|
// // }
|
||||||
.create_file(
|
// // };
|
||||||
"",
|
|
||||||
RUSTFS_META_TMP_BUCKET,
|
|
||||||
&format!("{}/{}/part.{}", tmp_id, dst_data_dir, part.number),
|
|
||||||
0,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
let writer = BitrotWriter::new(
|
|
||||||
Writer::from_tokio_writer(f),
|
|
||||||
erasure.shard_size(),
|
|
||||||
HashAlgorithm::HighwayHash256,
|
|
||||||
);
|
|
||||||
writers.push(Some(writer));
|
|
||||||
}
|
|
||||||
|
|
||||||
// let writer = new_bitrot_filewriter(
|
// if is_inline_buffer {
|
||||||
// disk.clone(),
|
// let writer = BitrotWriter::new(
|
||||||
// RUSTFS_META_TMP_BUCKET,
|
// Writer::from_cursor(Cursor::new(Vec::new())),
|
||||||
// format!("{}/{}/part.{}", tmp_id, dst_data_dir, part.number).as_str(),
|
// erasure.shard_size(),
|
||||||
// is_inline_buffer,
|
// HashAlgorithm::HighwayHash256,
|
||||||
// DEFAULT_BITROT_ALGO,
|
// );
|
||||||
// erasure.shard_size(erasure.block_size),
|
// writers.push(Some(writer));
|
||||||
// )
|
// } else {
|
||||||
// .await?;
|
// let f = disk
|
||||||
|
// .create_file(
|
||||||
|
// "",
|
||||||
|
// RUSTFS_META_TMP_BUCKET,
|
||||||
|
// &format!("{}/{}/part.{}", tmp_id, dst_data_dir, part.number),
|
||||||
|
// 0,
|
||||||
|
// )
|
||||||
|
// .await?;
|
||||||
|
// let writer = BitrotWriter::new(
|
||||||
|
// Writer::from_tokio_writer(f),
|
||||||
|
// erasure.shard_size(),
|
||||||
|
// HashAlgorithm::HighwayHash256,
|
||||||
|
// );
|
||||||
|
// writers.push(Some(writer));
|
||||||
|
// }
|
||||||
|
|
||||||
// writers.push(Some(writer));
|
// // let writer = new_bitrot_filewriter(
|
||||||
} else {
|
// // disk.clone(),
|
||||||
writers.push(None);
|
// // RUSTFS_META_TMP_BUCKET,
|
||||||
}
|
// // format!("{}/{}/part.{}", tmp_id, dst_data_dir, part.number).as_str(),
|
||||||
|
// // is_inline_buffer,
|
||||||
|
// // DEFAULT_BITROT_ALGO,
|
||||||
|
// // erasure.shard_size(erasure.block_size),
|
||||||
|
// // )
|
||||||
|
// // .await?;
|
||||||
|
|
||||||
|
// // writers.push(Some(writer));
|
||||||
|
// } else {
|
||||||
|
// writers.push(None);
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Heal each part. erasure.Heal() will write the healed
|
// Heal each part. erasure.Heal() will write the healed
|
||||||
@@ -2630,8 +2596,7 @@ impl SetDisks {
|
|||||||
// if let Some(w) = writer.as_any().downcast_ref::<BitrotFileWriter>() {
|
// if let Some(w) = writer.as_any().downcast_ref::<BitrotFileWriter>() {
|
||||||
// parts_metadata[index].data = Some(w.inline_data().to_vec());
|
// parts_metadata[index].data = Some(w.inline_data().to_vec());
|
||||||
// }
|
// }
|
||||||
parts_metadata[index].data =
|
parts_metadata[index].data = Some(writer.into_inline_data().unwrap_or_default());
|
||||||
Some(writer.into_inner().into_cursor_inner().unwrap_or_default());
|
|
||||||
}
|
}
|
||||||
parts_metadata[index].set_inline_data();
|
parts_metadata[index].set_inline_data();
|
||||||
} else {
|
} else {
|
||||||
@@ -3882,27 +3847,38 @@ impl ObjectIO for SetDisks {
|
|||||||
let mut errors = Vec::with_capacity(shuffle_disks.len());
|
let mut errors = Vec::with_capacity(shuffle_disks.len());
|
||||||
for disk_op in shuffle_disks.iter() {
|
for disk_op in shuffle_disks.iter() {
|
||||||
if let Some(disk) = disk_op {
|
if let Some(disk) = disk_op {
|
||||||
let writer = if is_inline_buffer {
|
let writer = create_bitrot_writer(
|
||||||
BitrotWriter::new(
|
is_inline_buffer,
|
||||||
Writer::from_cursor(Cursor::new(Vec::new())),
|
Some(disk),
|
||||||
erasure.shard_size(),
|
RUSTFS_META_TMP_BUCKET,
|
||||||
HashAlgorithm::HighwayHash256,
|
&tmp_object,
|
||||||
)
|
erasure.shard_file_size(data.content_length),
|
||||||
} else {
|
erasure.shard_size(),
|
||||||
let f = match disk
|
HashAlgorithm::HighwayHash256,
|
||||||
.create_file("", RUSTFS_META_TMP_BUCKET, &tmp_object, erasure.shard_file_size(data.content_length))
|
)
|
||||||
.await
|
.await?;
|
||||||
{
|
|
||||||
Ok(f) => f,
|
|
||||||
Err(e) => {
|
|
||||||
errors.push(Some(e));
|
|
||||||
writers.push(None);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
BitrotWriter::new(Writer::from_tokio_writer(f), erasure.shard_size(), HashAlgorithm::HighwayHash256)
|
// let writer = if is_inline_buffer {
|
||||||
};
|
// BitrotWriter::new(
|
||||||
|
// Writer::from_cursor(Cursor::new(Vec::new())),
|
||||||
|
// erasure.shard_size(),
|
||||||
|
// HashAlgorithm::HighwayHash256,
|
||||||
|
// )
|
||||||
|
// } else {
|
||||||
|
// let f = match disk
|
||||||
|
// .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_object, erasure.shard_file_size(data.content_length))
|
||||||
|
// .await
|
||||||
|
// {
|
||||||
|
// Ok(f) => f,
|
||||||
|
// Err(e) => {
|
||||||
|
// errors.push(Some(e));
|
||||||
|
// writers.push(None);
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
// };
|
||||||
|
|
||||||
|
// BitrotWriter::new(Writer::from_tokio_writer(f), erasure.shard_size(), HashAlgorithm::HighwayHash256)
|
||||||
|
// };
|
||||||
|
|
||||||
writers.push(Some(writer));
|
writers.push(Some(writer));
|
||||||
errors.push(None);
|
errors.push(None);
|
||||||
@@ -3952,7 +3928,7 @@ impl ObjectIO for SetDisks {
|
|||||||
for (i, fi) in parts_metadatas.iter_mut().enumerate() {
|
for (i, fi) in parts_metadatas.iter_mut().enumerate() {
|
||||||
if is_inline_buffer {
|
if is_inline_buffer {
|
||||||
if let Some(writer) = writers[i].take() {
|
if let Some(writer) = writers[i].take() {
|
||||||
fi.data = Some(writer.into_inner().into_cursor_inner().unwrap_or_default());
|
fi.data = Some(writer.into_inline_data().unwrap_or_default());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4553,21 +4529,32 @@ impl StorageAPI for SetDisks {
|
|||||||
let mut errors = Vec::with_capacity(shuffle_disks.len());
|
let mut errors = Vec::with_capacity(shuffle_disks.len());
|
||||||
for disk_op in shuffle_disks.iter() {
|
for disk_op in shuffle_disks.iter() {
|
||||||
if let Some(disk) = disk_op {
|
if let Some(disk) = disk_op {
|
||||||
let writer = {
|
let writer = create_bitrot_writer(
|
||||||
let f = match disk
|
false,
|
||||||
.create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, erasure.shard_file_size(data.content_length))
|
Some(disk),
|
||||||
.await
|
RUSTFS_META_TMP_BUCKET,
|
||||||
{
|
&tmp_part_path,
|
||||||
Ok(f) => f,
|
erasure.shard_file_size(data.content_length),
|
||||||
Err(e) => {
|
erasure.shard_size(),
|
||||||
errors.push(Some(e));
|
HashAlgorithm::HighwayHash256,
|
||||||
writers.push(None);
|
)
|
||||||
continue;
|
.await?;
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
BitrotWriter::new(Writer::from_tokio_writer(f), erasure.shard_size(), HashAlgorithm::HighwayHash256)
|
// let writer = {
|
||||||
};
|
// let f = match disk
|
||||||
|
// .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, erasure.shard_file_size(data.content_length))
|
||||||
|
// .await
|
||||||
|
// {
|
||||||
|
// Ok(f) => f,
|
||||||
|
// Err(e) => {
|
||||||
|
// errors.push(Some(e));
|
||||||
|
// writers.push(None);
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
// };
|
||||||
|
|
||||||
|
// BitrotWriter::new(Writer::from_tokio_writer(f), erasure.shard_size(), HashAlgorithm::HighwayHash256)
|
||||||
|
// };
|
||||||
|
|
||||||
writers.push(Some(writer));
|
writers.push(Some(writer));
|
||||||
errors.push(None);
|
errors.push(None);
|
||||||
@@ -6079,7 +6066,6 @@ mod tests {
|
|||||||
// Test object directory dangling detection
|
// Test object directory dangling detection
|
||||||
let errs = vec![Some(DiskError::FileNotFound), Some(DiskError::FileNotFound), None];
|
let errs = vec![Some(DiskError::FileNotFound), Some(DiskError::FileNotFound), None];
|
||||||
assert!(is_object_dir_dang_ling(&errs));
|
assert!(is_object_dir_dang_ling(&errs));
|
||||||
|
|
||||||
let errs2 = vec![None, None, None];
|
let errs2 = vec![None, None, None];
|
||||||
assert!(!is_object_dir_dang_ling(&errs2));
|
assert!(!is_object_dir_dang_ling(&errs2));
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user