feat(storage): add direct chunk GET fast path (#2351)

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
This commit is contained in:
houseme
2026-04-07 08:33:46 +08:00
committed by GitHub
parent 8d27170ce4
commit 32bf8f5bf3
84 changed files with 15932 additions and 3592 deletions
+28
View File
@@ -49,6 +49,34 @@ pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE";
/// to regular I/O without errors.
pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = true;
/// Environment variable for zero-copy read operating mode.
///
/// Supported values:
/// - `off`: disable mmap-backed chunk fast path and always use the compatibility path
/// - `conservative`: allow a single mmap window per request
/// - `balanced`: allow multiple mmap windows with the default size guardrails
/// - `aggressive`: allow multi-window mmap and relax the small-object cutoff
pub const ENV_OBJECT_ZERO_COPY_MODE: &str = "RUSTFS_OBJECT_ZERO_COPY_MODE";
/// Default zero-copy read mode.
pub const DEFAULT_OBJECT_ZERO_COPY_MODE: &str = "balanced";
/// Environment variable for the maximum mmap window size used by the chunk fast path.
///
/// This controls the visible bytes per mapped chunk before the implementation emits a new window.
pub const ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES";
/// Default mmap window size for chunk fast path reads: 8 MiB.
pub const DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: usize = 8 * 1024 * 1024;
/// Environment variable for the maximum total active mmap bytes.
///
/// Requests that would exceed this active window budget fall back to the compatibility path.
pub const ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES";
/// Default maximum active mmap bytes across concurrent local chunk fast-path reads: 256 MiB.
pub const DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: usize = 256 * 1024 * 1024;
// =============================================================================
// Direct I/O Configuration
// =============================================================================
+8
View File
@@ -20,6 +20,11 @@ license.workspace = true
repository.workspace = true
rust-version.workspace = true
[[bin]]
name = "small_put_bench"
path = "src/bin/small_put_bench.rs"
test = false
[lints]
workspace = true
@@ -41,6 +46,7 @@ serde_json.workspace = true
tonic = { workspace = true }
tokio = { workspace = true }
tokio-stream = { workspace = true }
clap = { workspace = true }
rustfs-madmin.workspace = true
rustfs-filemeta.workspace = true
bytes.workspace = true
@@ -52,6 +58,8 @@ async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
async-trait = { workspace = true }
flate2.workspace = true
http.workspace = true
http-body.workspace = true
http-body-util.workspace = true
reqwest = { workspace = true }
rustfs-signer.workspace = true
tracing = { workspace = true }
+441
View File
@@ -0,0 +1,441 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use anyhow::{Context, Result, anyhow, bail};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{Delete, ObjectIdentifier};
use aws_sdk_s3::{Client, Config};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use bytes::Bytes;
use clap::Parser;
use serde::Serialize;
use std::path::PathBuf;
use std::time::{Duration, Instant};
#[derive(Parser, Debug)]
#[command(name = "small_put_bench")]
#[command(about = "Rust-native small PUT benchmark for RustFS-compatible S3 endpoints")]
struct Args {
#[arg(long, env = "RUSTFS_BENCH_ENDPOINT")]
endpoint: String,
#[arg(long, env = "RUSTFS_BENCH_ACCESS_KEY", default_value = "rustfsadmin")]
access_key: String,
#[arg(long, env = "RUSTFS_BENCH_SECRET_KEY", default_value = "rustfsadmin")]
secret_key: String,
#[arg(long, env = "RUSTFS_BENCH_REGION", default_value = "us-east-1")]
region: String,
#[arg(long, env = "RUSTFS_BENCH_BUCKET", default_value = "small-put-benchmark")]
bucket: String,
#[arg(long, env = "RUSTFS_BENCH_SIZES", default_value = "4KiB,16KiB,64KiB,256KiB,1MiB")]
sizes: String,
#[arg(long, env = "RUSTFS_BENCH_CONCURRENCY", default_value_t = 8)]
concurrency: usize,
#[arg(long, env = "RUSTFS_BENCH_DURATION_SECS", default_value_t = 10)]
duration_secs: u64,
#[arg(long, env = "RUSTFS_BENCH_TIMEOUT_SECS", default_value_t = 15)]
timeout_secs: u64,
#[arg(long, env = "RUSTFS_BENCH_PREFIX")]
prefix: Option<String>,
#[arg(long)]
output_json: Option<PathBuf>,
#[arg(long, default_value_t = false)]
cleanup: bool,
}
#[derive(Clone, Debug)]
struct SizeSpec {
label: String,
slug: String,
bytes: usize,
}
#[derive(Debug)]
struct Sample {
ok: bool,
duration_ms: f64,
}
#[derive(Debug, Serialize)]
struct SizeSummary {
label: String,
bytes: usize,
total: usize,
succeeded: usize,
failed: usize,
wall_secs: f64,
object_rate: f64,
throughput_mib_per_sec: f64,
avg_ms: Option<f64>,
p50_ms: Option<f64>,
p90_ms: Option<f64>,
p99_ms: Option<f64>,
}
#[derive(Debug, Serialize)]
struct RunSummary {
run_id: String,
endpoint: String,
bucket: String,
concurrency: usize,
duration_secs: u64,
timeout_secs: u64,
sizes: Vec<SizeSummary>,
}
fn main() -> Result<()> {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.context("failed to build tokio runtime")?;
runtime.block_on(async_main())
}
async fn async_main() -> Result<()> {
let args = Args::parse();
validate_args(&args)?;
let sizes = parse_size_list(&args.sizes)?;
let run_id = args.prefix.clone().unwrap_or_else(default_run_id);
let client = build_s3_client(&args.endpoint, &args.access_key, &args.secret_key, &args.region);
ensure_bucket(&client, &args.bucket).await?;
let mut size_summaries = Vec::with_capacity(sizes.len());
for size in &sizes {
let summary = run_size_benchmark(
client.clone(),
args.bucket.clone(),
run_id.clone(),
size.clone(),
args.concurrency,
Duration::from_secs(args.duration_secs),
Duration::from_secs(args.timeout_secs),
)
.await?;
print_size_summary(&summary);
size_summaries.push(summary);
}
if args.cleanup {
cleanup_prefix(&client, &args.bucket, &run_id).await?;
}
let summary = RunSummary {
run_id,
endpoint: args.endpoint,
bucket: args.bucket,
concurrency: args.concurrency,
duration_secs: args.duration_secs,
timeout_secs: args.timeout_secs,
sizes: size_summaries,
};
if let Some(path) = args.output_json {
let json = serde_json::to_vec_pretty(&summary).context("failed to serialize benchmark summary")?;
std::fs::write(&path, json).with_context(|| format!("failed to write benchmark summary to {}", path.display()))?;
println!("Wrote summary to {}", path.display());
}
Ok(())
}
fn validate_args(args: &Args) -> Result<()> {
if args.concurrency == 0 {
bail!("--concurrency must be greater than zero");
}
if args.duration_secs == 0 {
bail!("--duration-secs must be greater than zero");
}
if args.timeout_secs == 0 {
bail!("--timeout-secs must be greater than zero");
}
Ok(())
}
fn build_s3_client(endpoint: &str, access_key: &str, secret_key: &str, region: &str) -> Client {
let credentials = Credentials::new(access_key, secret_key, None, None, "small-put-bench");
let mut config = Config::builder()
.credentials_provider(credentials)
.region(Region::new(region.to_string()))
.endpoint_url(endpoint)
.force_path_style(true)
.behavior_version_latest();
if endpoint.starts_with("http://") {
config = config.http_client(SmithyHttpClientBuilder::new().build_http());
}
Client::from_conf(config.build())
}
async fn ensure_bucket(client: &Client, bucket: &str) -> Result<()> {
if client.head_bucket().bucket(bucket).send().await.is_ok() {
return Ok(());
}
match client.create_bucket().bucket(bucket).send().await {
Ok(_) => Ok(()),
Err(err) => {
let rendered = err.to_string();
if rendered.contains("BucketAlreadyOwnedByYou") || rendered.contains("BucketAlreadyExists") {
Ok(())
} else {
Err(err).with_context(|| format!("failed to create benchmark bucket {bucket}"))
}
}
}
}
async fn run_size_benchmark(
client: Client,
bucket: String,
run_id: String,
size: SizeSpec,
concurrency: usize,
duration: Duration,
timeout: Duration,
) -> Result<SizeSummary> {
let payload = Bytes::from(vec![0_u8; size.bytes]);
let deadline = Instant::now() + duration;
let wall_start = Instant::now();
let mut handles = Vec::with_capacity(concurrency);
for worker in 0..concurrency {
let client = client.clone();
let bucket = bucket.clone();
let payload = payload.clone();
let prefix = format!("{run_id}/{}/worker-{worker}", size.slug);
handles.push(tokio::spawn(async move {
let mut samples = Vec::new();
let mut idx = 0usize;
while Instant::now() < deadline {
let key = format!("{prefix}/obj-{idx}.bin");
let started_at = Instant::now();
let request = client
.put_object()
.bucket(&bucket)
.key(key)
.body(ByteStream::from(payload.clone()))
.content_type("application/octet-stream");
let ok = matches!(tokio::time::timeout(timeout, request.send()).await, Ok(Ok(_)));
samples.push(Sample {
ok,
duration_ms: started_at.elapsed().as_secs_f64() * 1000.0,
});
idx += 1;
}
samples
}));
}
let mut samples = Vec::new();
for handle in handles {
samples.extend(handle.await.map_err(|err| anyhow!("benchmark worker join error: {err}"))?);
}
Ok(build_size_summary(&size, samples, wall_start.elapsed()))
}
fn build_size_summary(size: &SizeSpec, mut samples: Vec<Sample>, wall_elapsed: Duration) -> SizeSummary {
let total = samples.len();
let succeeded = samples.iter().filter(|sample| sample.ok).count();
let failed = total.saturating_sub(succeeded);
let wall_secs = wall_elapsed.as_secs_f64();
let object_rate = if wall_secs > 0.0 { succeeded as f64 / wall_secs } else { 0.0 };
let throughput_mib_per_sec = if wall_secs > 0.0 {
((size.bytes * succeeded) as f64 / (1024.0 * 1024.0)) / wall_secs
} else {
0.0
};
let avg_ms = if total > 0 {
Some(samples.iter().map(|sample| sample.duration_ms).sum::<f64>() / total as f64)
} else {
None
};
samples.sort_by(|lhs, rhs| lhs.duration_ms.total_cmp(&rhs.duration_ms));
let durations: Vec<f64> = samples.into_iter().map(|sample| sample.duration_ms).collect();
SizeSummary {
label: size.label.clone(),
bytes: size.bytes,
total,
succeeded,
failed,
wall_secs,
object_rate,
throughput_mib_per_sec,
avg_ms,
p50_ms: percentile(&durations, 0.50),
p90_ms: percentile(&durations, 0.90),
p99_ms: percentile(&durations, 0.99),
}
}
async fn cleanup_prefix(client: &Client, bucket: &str, prefix: &str) -> Result<()> {
let mut continuation_token = None;
loop {
let response = client
.list_objects_v2()
.bucket(bucket)
.prefix(prefix)
.set_continuation_token(continuation_token.clone())
.send()
.await
.with_context(|| format!("failed to list objects for cleanup under {bucket}/{prefix}"))?;
let objects: Vec<ObjectIdentifier> = response
.contents
.unwrap_or_default()
.into_iter()
.filter_map(|object| object.key.map(|key| ObjectIdentifier::builder().key(key).build().ok()))
.flatten()
.collect();
for chunk in objects.chunks(1_000) {
if chunk.is_empty() {
continue;
}
client
.delete_objects()
.bucket(bucket)
.delete(
Delete::builder()
.set_objects(Some(chunk.to_vec()))
.quiet(true)
.build()
.context("failed to build delete request")?,
)
.send()
.await
.with_context(|| format!("failed to delete cleanup batch under {bucket}/{prefix}"))?;
}
if response.is_truncated.unwrap_or(false) {
continuation_token = response.next_continuation_token;
} else {
break;
}
}
Ok(())
}
fn parse_size_list(input: &str) -> Result<Vec<SizeSpec>> {
input
.split(',')
.map(str::trim)
.filter(|item| !item.is_empty())
.map(parse_size_spec)
.collect()
}
fn parse_size_spec(input: &str) -> Result<SizeSpec> {
let normalized = input.trim();
let lower = normalized.to_ascii_lowercase();
let (number_part, multiplier) = if let Some(value) = lower.strip_suffix("kib") {
(value, 1024usize)
} else if let Some(value) = lower.strip_suffix("mib") {
(value, 1024usize * 1024usize)
} else if let Some(value) = lower.strip_suffix('b') {
(value, 1usize)
} else {
(lower.as_str(), 1usize)
};
let value = number_part
.trim()
.parse::<usize>()
.with_context(|| format!("invalid size component: {input}"))?;
let bytes = value
.checked_mul(multiplier)
.ok_or_else(|| anyhow!("size overflow for {input}"))?;
Ok(SizeSpec {
label: normalized.to_string(),
slug: normalized
.chars()
.filter(|ch| ch.is_ascii_alphanumeric())
.collect::<String>()
.to_ascii_lowercase(),
bytes,
})
}
fn percentile(values: &[f64], percentile: f64) -> Option<f64> {
if values.is_empty() {
return None;
}
let index = ((values.len() - 1) as f64 * percentile).floor() as usize;
values.get(index).copied()
}
fn default_run_id() -> String {
format!("small-put-bench-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"))
}
fn print_size_summary(summary: &SizeSummary) {
println!(
"{}: success={} failed={} obj/s={:.3} MiB/s={:.3} avg={:.3?} p50={:.3?} p90={:.3?} p99={:.3?}",
summary.label,
summary.succeeded,
summary.failed,
summary.object_rate,
summary.throughput_mib_per_sec,
summary.avg_ms,
summary.p50_ms,
summary.p90_ms,
summary.p99_ms,
);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_size_spec_supports_binary_units() {
let four_kib = parse_size_spec("4KiB").expect("4KiB should parse");
assert_eq!(four_kib.bytes, 4 * 1024);
let one_mib = parse_size_spec("1MiB").expect("1MiB should parse");
assert_eq!(one_mib.bytes, 1024 * 1024);
}
#[test]
fn percentile_returns_expected_bucket() {
let values = vec![10.0, 20.0, 30.0, 40.0, 50.0];
assert_eq!(percentile(&values, 0.50), Some(30.0));
assert_eq!(percentile(&values, 0.90), Some(40.0));
}
}
+275 -1
View File
@@ -19,9 +19,13 @@
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::primitives::{ByteStream, SdkBody};
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
use base64::Engine;
use bytes::Bytes;
use futures::StreamExt;
use http_body::Frame;
use http_body_util::StreamBody;
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
use serial_test::serial;
use sha2::{Digest, Sha256};
@@ -64,6 +68,53 @@ mod tests {
.encoded
}
fn streamed_body_70kib_of_a() -> ByteStream {
let bytes = Bytes::from_static(&[b'a'; 1024]);
let stream = futures::stream::repeat_with(move || {
let frame = Frame::data(bytes.clone());
Ok::<_, std::io::Error>(frame)
});
let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024);
ByteStream::new(SdkBody::from_body_1_x(body))
}
struct WithSizeHint<T> {
inner: T,
size_hint: usize,
}
impl<T> WithSizeHint<T> {
fn new(inner: T, size_hint: usize) -> Self {
Self { inner, size_hint }
}
}
impl<T> http_body::Body for WithSizeHint<T>
where
T: http_body::Body + Unpin,
{
type Data = T::Data;
type Error = T::Error;
fn poll_frame(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
let this = self.get_mut();
std::pin::Pin::new(&mut this.inner).poll_frame(cx)
}
fn is_end_stream(&self) -> bool {
self.inner.is_end_stream()
}
fn size_hint(&self) -> http_body::SizeHint {
let mut hint = self.inner.size_hint();
hint.set_exact(self.size_hint as u64);
hint
}
}
/// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
#[tokio::test]
#[serial]
@@ -136,6 +187,121 @@ mod tests {
info!("PASSED: PutObject with checksum_sha256 and GetObject content match");
}
/// Mirrors `s3s-e2e` behavior: only request `checksum_algorithm`, then expect
/// both PutObject and GetObject(checksum_mode=enabled) to expose the same checksum.
#[tokio::test]
#[serial]
async fn test_put_object_with_checksum_algorithm_only() {
init_logging();
info!("TEST: PutObject with checksum_algorithm only");
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = create_s3_client(&env);
let bucket = "test-checksum-algorithm-only";
create_bucket(&client, bucket).await.expect("Failed to create bucket");
let key = "obj-with-checksum-algorithm-only.txt";
let content = vec![b'a'; 70 * 1024];
let put_resp = client
.put_object()
.bucket(bucket)
.key(key)
.checksum_algorithm(ChecksumAlgorithm::Crc32)
.body(ByteStream::from(content.clone()))
.send()
.await
.expect("PutObject with checksum_algorithm should succeed");
let put_checksum = put_resp
.checksum_crc32()
.expect("PutObject should return checksum_crc32 when checksum_algorithm is used")
.to_string();
let mut get_resp = client
.get_object()
.bucket(bucket)
.key(key)
.checksum_mode(ChecksumMode::Enabled)
.send()
.await
.expect("GetObject should succeed");
let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(aws_sdk_s3::primitives::SdkBody::empty()))
.collect()
.await
.expect("collect body")
.into_bytes();
assert_eq!(body_bytes.as_ref(), content.as_slice(), "GetObject body must match uploaded content");
assert_eq!(
get_resp.checksum_crc32().map(str::to_string),
Some(put_checksum),
"GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum"
);
}
/// Matches the `s3s-e2e` streaming upload shape more closely than `ByteStream::from(Vec<u8>)`.
#[tokio::test]
#[serial]
async fn test_put_object_with_checksum_algorithm_only_streaming_body() {
init_logging();
info!("TEST: PutObject with checksum_algorithm only using streaming body");
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = create_s3_client(&env);
let bucket = "test-checksum-algorithm-streaming";
create_bucket(&client, bucket).await.expect("Failed to create bucket");
let key = "obj-with-checksum-algorithm-streaming.txt";
let expected_content = vec![b'a'; 70 * 1024];
let put_resp = client
.put_object()
.bucket(bucket)
.key(key)
.checksum_algorithm(ChecksumAlgorithm::Crc32)
.body(streamed_body_70kib_of_a())
.send()
.await
.expect("PutObject with streaming checksum_algorithm should succeed");
let put_checksum = put_resp
.checksum_crc32()
.expect("PutObject should return checksum_crc32 for streaming checksum_algorithm uploads")
.to_string();
let mut get_resp = client
.get_object()
.bucket(bucket)
.key(key)
.checksum_mode(ChecksumMode::Enabled)
.send()
.await
.expect("GetObject should succeed");
let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(SdkBody::empty()))
.collect()
.await
.expect("collect body")
.into_bytes();
assert_eq!(
body_bytes.as_ref(),
expected_content.as_slice(),
"GetObject body must match uploaded content"
);
assert_eq!(
get_resp.checksum_crc32().map(str::to_string),
Some(put_checksum),
"GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum for streaming uploads"
);
}
/// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
/// Uses part size >= 5MB (server minimum) for two parts.
#[tokio::test]
@@ -234,6 +400,114 @@ mod tests {
info!("PASSED: MultipartUpload with checksum and GetObject content match");
}
/// Mirrors `s3s-e2e` multipart behavior: request checksum algorithm at MPU creation,
/// rely on auto checksum handling during UploadPart, and expect CompleteMultipartUpload to succeed.
#[tokio::test]
#[serial]
async fn test_multipart_upload_with_crc32_algorithm_only() {
init_logging();
info!("TEST: MultipartUpload with checksum_algorithm only (CRC32)");
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = create_s3_client(&env);
let bucket = "test-multipart-checksum-crc32-auto";
create_bucket(&client, bucket).await.expect("Failed to create bucket");
let key = "multipart-with-crc32-auto.bin";
let part1_content = "a".repeat(5 * 1024 * 1024 + 1);
let part2_content = "b".repeat(1024);
let create_resp = client
.create_multipart_upload()
.bucket(bucket)
.key(key)
.checksum_algorithm(ChecksumAlgorithm::Crc32)
.send()
.await
.expect("CreateMultipartUpload should succeed");
let upload_id = create_resp.upload_id().expect("upload_id should be present");
let part1_resp = client
.upload_part()
.bucket(bucket)
.key(key)
.upload_id(upload_id)
.part_number(1)
.body(ByteStream::from(part1_content.clone().into_bytes()))
.send()
.await
.expect("UploadPart 1 should succeed");
let part1_checksum = part1_resp
.checksum_crc32()
.expect("UploadPart 1 should return checksum_crc32")
.to_string();
let part2_resp = client
.upload_part()
.bucket(bucket)
.key(key)
.upload_id(upload_id)
.part_number(2)
.body(ByteStream::from(part2_content.clone().into_bytes()))
.send()
.await
.expect("UploadPart 2 should succeed");
let part2_checksum = part2_resp
.checksum_crc32()
.expect("UploadPart 2 should return checksum_crc32")
.to_string();
let completed_upload = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(part1_resp.e_tag().expect("etag part 1"))
.checksum_crc32(part1_checksum)
.build(),
)
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(part2_resp.e_tag().expect("etag part 2"))
.checksum_crc32(part2_checksum)
.build(),
)
.build();
client
.complete_multipart_upload()
.bucket(bucket)
.key(key)
.upload_id(upload_id)
.multipart_upload(completed_upload)
.send()
.await
.expect("CompleteMultipartUpload should succeed");
let body_bytes = client
.get_object()
.bucket(bucket)
.key(key)
.send()
.await
.expect("GetObject should succeed")
.body
.collect()
.await
.expect("collect body")
.into_bytes();
let expected_content = format!("{part1_content}{part2_content}");
assert_eq!(
body_bytes.as_ref(),
expected_content.as_bytes(),
"completed multipart object must match concatenated parts"
);
}
/// Regression test for issue #2282:
/// CRC64NVME full-object checksum should match between direct PutObject and multipart upload.
#[tokio::test]
+96
View File
@@ -344,6 +344,9 @@ async fn test_local_kms_multipart_upload() {
test_multipart_upload_with_sse_c(&s3_client, TEST_BUCKET)
.await
.expect("SSE-C multipart upload test failed");
test_multipart_download_with_wrong_sse_c_key_fails(&s3_client, TEST_BUCKET)
.await
.expect("SSE-C multipart wrong-key download test failed");
// Test 4: Large multipart upload (test streaming encryption with multiple blocks)
// TODO: Re-enable after fixing streaming encryption issues with large files
@@ -648,6 +651,99 @@ async fn test_multipart_upload_with_sse_c(
Ok(())
}
async fn test_multipart_download_with_wrong_sse_c_key_fails(
s3_client: &aws_sdk_s3::Client,
bucket: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let object_key = "multipart-sse-c-bad-download-test";
let part_size = 5 * 1024 * 1024;
let total_parts = 2;
let total_size = part_size * total_parts;
let encryption_key = "01234567890123456789012345678901";
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
let key_md5 = sse_customer_key_md5_base64(encryption_key);
let wrong_key = "abcdefghijklmnopqrstuvwxyz012345";
let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 5) % 256) as u8).collect();
let create_multipart_output = s3_client
.create_multipart_upload()
.bucket(bucket)
.key(object_key)
.sse_customer_algorithm("AES256")
.sse_customer_key(&key_b64)
.sse_customer_key_md5(&key_md5)
.send()
.await?;
let upload_id = create_multipart_output.upload_id().unwrap();
let mut completed_parts = Vec::new();
for part_number in 1..=total_parts {
let start = (part_number - 1) * part_size;
let end = std::cmp::min(start + part_size, total_size);
let part_data = &test_data[start..end];
let upload_part_output = s3_client
.upload_part()
.bucket(bucket)
.key(object_key)
.upload_id(upload_id)
.part_number(part_number as i32)
.body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec()))
.sse_customer_algorithm("AES256")
.sse_customer_key(&key_b64)
.sse_customer_key_md5(&key_md5)
.send()
.await?;
completed_parts.push(
aws_sdk_s3::types::CompletedPart::builder()
.part_number(part_number as i32)
.e_tag(upload_part_output.e_tag().unwrap())
.build(),
);
}
let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
.set_parts(Some(completed_parts))
.build();
s3_client
.complete_multipart_upload()
.bucket(bucket)
.key(object_key)
.upload_id(upload_id)
.multipart_upload(completed_multipart_upload)
.send()
.await?;
let err = s3_client
.get_object()
.bucket(bucket)
.key(object_key)
.sse_customer_algorithm("AES256")
.sse_customer_key(&wrong_key_b64)
.sse_customer_key_md5(&wrong_key_md5)
.send()
.await
.expect_err("multipart SSE-C download with the wrong key should fail");
let service_err = err.into_service_error();
assert_eq!(
service_err.meta().code(),
Some("InvalidRequest"),
"wrong-key multipart SSE-C download should return InvalidRequest, got {:?}",
service_err.meta().code()
);
Ok(())
}
/// Test large multipart upload to verify streaming encryption works correctly
#[allow(dead_code)]
async fn test_large_multipart_upload(
+4
View File
@@ -97,6 +97,10 @@ mod cluster_concurrency_test;
#[cfg(test)]
mod checksum_upload_test;
// Range request regression tests
#[cfg(test)]
mod range_request_test;
// Group deletion tests
#[cfg(test)]
mod group_delete_test;
+80
View File
@@ -0,0 +1,80 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! End-to-end regression test for invalid GET object ranges.
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::primitives::ByteStream;
use serial_test::serial;
use tracing::info;
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
env.create_s3_client()
}
async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
match client.create_bucket().bucket(bucket).send().await {
Ok(_) => Ok(()),
Err(err) => {
if err.to_string().contains("BucketAlreadyOwnedByYou") || err.to_string().contains("BucketAlreadyExists") {
Ok(())
} else {
Err(Box::new(err))
}
}
}
}
#[tokio::test]
#[serial]
async fn test_get_object_invalid_range_returns_416_issue_s3_implemented_tests() {
init_logging();
info!("TEST: GetObject invalid range should return InvalidRange/416");
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = create_s3_client(&env);
let bucket = "test-invalid-range";
let key = "range.txt";
let content = b"testcontent";
create_bucket(&client, bucket).await.expect("Failed to create bucket");
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(content))
.send()
.await
.expect("PutObject should succeed");
let result = client.get_object().bucket(bucket).key(key).range("bytes=40-50").send().await;
let err = result.expect_err("GetObject with an unsatisfiable range should fail");
match err {
SdkError::ServiceError(service_err) => {
assert_eq!(service_err.raw().status().as_u16(), 416, "invalid range should return HTTP 416");
let s3_err = service_err.into_err();
assert_eq!(s3_err.meta().code(), Some("InvalidRange"), "invalid range should map to InvalidRange");
}
other_err => panic!("Expected S3 service error, got: {other_err:?}"),
}
}
}
+14
View File
@@ -70,6 +70,7 @@ reed-solomon-erasure = { workspace = true }
reed-solomon-simd = { workspace = true }
lazy_static.workspace = true
rustfs-lock.workspace = true
rustfs-io-core.workspace = true
rustfs-io-metrics.workspace = true
regex = { workspace = true }
path-absolutize = { workspace = true }
@@ -97,6 +98,7 @@ rand.workspace = true
pin-project-lite.workspace = true
md-5.workspace = true
memmap2 = { workspace = true }
libc.workspace = true
rustfs-madmin.workspace = true
rustfs-workers.workspace = true
reqwest = { workspace = true }
@@ -138,5 +140,17 @@ harness = false
name = "comparison_benchmark"
harness = false
[[bench]]
name = "direct_chunk_benchmark"
harness = false
[[bench]]
name = "reconstructed_chunk_benchmark"
harness = false
[[bench]]
name = "bitrot_chunk_benchmark"
harness = false
[lib]
doctest = false
+37
View File
@@ -32,6 +32,43 @@
For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
## 📈 Benchmarks
ECStore ships several Criterion benchmarks under [`crates/ecstore/benches/`](./benches/).
### Direct Chunk Path
Use the direct chunk benchmark to compare the current slice-forwarding path against the previous assembled-copy path:
```bash
cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark
```
To run only the end-to-end ECStore range-read benchmark:
```bash
cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark ecstore_get_object_chunks
```
To run the reconstructed multi-disk range-read benchmark:
```bash
cargo bench -p rustfs-ecstore --bench reconstructed_chunk_benchmark
```
### Saved Comparison Points
Latest local measurements on this branch:
- `direct_chunk_path/slice_forwarding/single_block_aligned`: about `477 ns`
- `direct_chunk_path/assembled_copy/single_block_aligned`: about `3.25 us`
- `direct_chunk_path/slice_forwarding/multi_block_unaligned`: about `963 ns`
- `direct_chunk_path/assembled_copy/multi_block_unaligned`: about `7.25 us`
- `ecstore_get_object_chunks/drain/multi_disk_range`: about `644-654 us`, throughput about `2.86-2.90 GiB/s`
- `reconstructed_chunk_path/drain/multi_disk_missing_shard`: about `1.292-1.304 ms`, throughput about `1.43-1.45 GiB/s`
These numbers are intended as branch-local reference points. Re-run the benchmark on your target machine before treating them as a regression baseline.
## 📄 License
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
@@ -0,0 +1,106 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use bytes::Bytes;
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use rustfs_ecstore::bitrot::decode_bitrot_chunk_source_for_bench;
use rustfs_io_core::IoChunk;
use rustfs_utils::HashAlgorithm;
use std::hint::black_box;
struct BitrotChunkBenchCase {
name: &'static str,
source_chunks: Vec<IoChunk>,
shard_size: usize,
expected_decoded_len: usize,
expected_copied: bool,
}
fn encode_shard(checksum_algo: HashAlgorithm, shard: &[u8]) -> Vec<u8> {
let mut encoded = Vec::with_capacity(checksum_algo.size() + shard.len());
encoded.extend_from_slice(checksum_algo.hash_encode(shard).as_ref());
encoded.extend_from_slice(shard);
encoded
}
fn bitrot_chunk_bench_cases() -> [BitrotChunkBenchCase; 2] {
let checksum_algo = HashAlgorithm::Md5;
let shard_one = b"abcd";
let shard_two = b"efgh";
let encoded_one = encode_shard(checksum_algo.clone(), shard_one);
let encoded_two = encode_shard(checksum_algo.clone(), shard_two);
let mut cross_chunk = Vec::with_capacity(encoded_one.len() + encoded_two.len());
cross_chunk.extend_from_slice(&encoded_one);
cross_chunk.extend_from_slice(&encoded_two);
let split = checksum_algo.size() + 2;
[
BitrotChunkBenchCase {
name: "aligned_multi_chunk_no_copy",
source_chunks: vec![
IoChunk::Shared(Bytes::from(encoded_one)),
IoChunk::Shared(Bytes::from(encoded_two)),
],
shard_size: shard_one.len(),
expected_decoded_len: shard_one.len() + shard_two.len(),
expected_copied: false,
},
BitrotChunkBenchCase {
name: "cross_chunk_frame_copy",
source_chunks: vec![
IoChunk::Shared(Bytes::copy_from_slice(&cross_chunk[..split])),
IoChunk::Shared(Bytes::copy_from_slice(&cross_chunk[split..])),
],
shard_size: shard_one.len(),
expected_decoded_len: shard_one.len() + shard_two.len(),
expected_copied: true,
},
]
}
fn bench_bitrot_chunk_decode(c: &mut Criterion) {
let checksum_algo = HashAlgorithm::Md5;
let mut group = c.benchmark_group("bitrot_chunk_decode");
group.sample_size(20);
for case in bitrot_chunk_bench_cases() {
let (decoded, copied) =
decode_bitrot_chunk_source_for_bench(&case.source_chunks, case.shard_size, checksum_algo.clone(), false)
.expect("decode bitrot source");
let decoded_len: usize = decoded.iter().map(IoChunk::len).sum();
assert_eq!(decoded_len, case.expected_decoded_len);
assert_eq!(copied, case.expected_copied);
group.throughput(Throughput::Bytes(case.expected_decoded_len as u64));
group.bench_with_input(BenchmarkId::new("decode", case.name), &case, |b, case| {
b.iter(|| {
let result = decode_bitrot_chunk_source_for_bench(
black_box(&case.source_chunks),
black_box(case.shard_size),
checksum_algo.clone(),
false,
)
.expect("decode bitrot source");
black_box(result);
});
});
}
group.finish();
}
criterion_group!(benches, bench_bitrot_chunk_decode);
criterion_main!(benches);
@@ -0,0 +1,390 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use bytes::{Bytes, BytesMut};
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use futures_util::StreamExt;
use futures_util::stream;
use http::HeaderMap;
use rustfs_ecstore::bucket::metadata_sys;
use rustfs_ecstore::disk::endpoint::Endpoint;
use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use rustfs_ecstore::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
use rustfs_ecstore::set_disk::collect_direct_data_shard_chunks_for_benchmark;
use rustfs_ecstore::store::{ECStore, init_local_disks};
use rustfs_ecstore::store_api::{
BucketOperations, BucketOptions, ChunkNativePutData, GetObjectChunkCopyMode, HTTPRangeSpec, MakeBucketOptions, ObjectIO,
ObjectOptions,
};
use rustfs_io_core::{BoxChunkStream, IoChunk, MappedChunk};
use std::hint::black_box;
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicU16, Ordering};
use tempfile::TempDir;
use tokio::runtime::Runtime;
use tokio_util::sync::CancellationToken;
#[derive(Clone)]
struct BenchCase {
name: &'static str,
data_shards: usize,
block_size: usize,
blocks: usize,
offset: usize,
length: usize,
}
fn bench_cases() -> [BenchCase; 2] {
[
BenchCase {
name: "single_block_aligned",
data_shards: 4,
block_size: 256 * 1024,
blocks: 1,
offset: 0,
length: 256 * 1024,
},
BenchCase {
name: "multi_block_unaligned",
data_shards: 4,
block_size: 256 * 1024,
blocks: 8,
offset: 123_457,
length: 2 * 256 * 1024 + 33_333,
},
]
}
#[derive(Clone)]
struct EcstoreBenchCase {
name: &'static str,
disk_count: usize,
payload_len: usize,
range: HTTPRangeSpec,
expected_copy_mode: GetObjectChunkCopyMode,
}
struct EcstoreBenchEnv {
_temp_dir: TempDir,
store: Arc<ECStore>,
bucket: String,
key: String,
range: HTTPRangeSpec,
opts: ObjectOptions,
expected_len: usize,
expected_copy_mode: GetObjectChunkCopyMode,
}
fn ecstore_bench_cases() -> [EcstoreBenchCase; 1] {
[EcstoreBenchCase {
name: "multi_disk_range",
disk_count: 4,
payload_len: 3 * 1024 * 1024 + 137,
range: HTTPRangeSpec {
is_suffix_length: false,
start: 123_457,
end: 2 * 1024 * 1024 + 33_333,
},
expected_copy_mode: expected_direct_copy_mode(),
}]
}
#[cfg(unix)]
const fn expected_direct_copy_mode() -> GetObjectChunkCopyMode {
GetObjectChunkCopyMode::TrueZeroCopy
}
#[cfg(not(unix))]
const fn expected_direct_copy_mode() -> GetObjectChunkCopyMode {
GetObjectChunkCopyMode::SharedBytes
}
fn clone_range_spec(range: &HTTPRangeSpec) -> HTTPRangeSpec {
HTTPRangeSpec {
is_suffix_length: range.is_suffix_length,
start: range.start,
end: range.end,
}
}
fn next_loopback_addr() -> SocketAddr {
static NEXT_PORT: AtomicU16 = AtomicU16::new(39013);
let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed);
SocketAddr::from(([127, 0, 0, 1], port))
}
fn build_endpoint_pools(paths: &[std::path::PathBuf]) -> EndpointServerPools {
let mut endpoints = Vec::with_capacity(paths.len());
for (idx, disk_path) in paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf8 path")).expect("endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(idx);
endpoints.push(endpoint);
}
EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: paths.len(),
endpoints: Endpoints::from(endpoints),
cmd_line: "bench".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
}])
}
async fn build_ecstore_bench_env(case: &EcstoreBenchCase) -> EcstoreBenchEnv {
let temp_dir = tempfile::tempdir().expect("tempdir");
let mut disk_paths = Vec::with_capacity(case.disk_count);
for idx in 0..case.disk_count {
let path = temp_dir.path().join(format!("disk{}", idx + 1));
tokio::fs::create_dir_all(&path).await.expect("create disk dir");
disk_paths.push(path);
}
let endpoint_pools = build_endpoint_pools(&disk_paths);
GLOBAL_LOCAL_DISK_MAP.write().await.clear();
GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
let store = ECStore::new(next_loopback_addr(), endpoint_pools, CancellationToken::new())
.await
.expect("create ecstore");
let buckets = store
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("list buckets")
.into_iter()
.map(|bucket| bucket.name)
.collect();
metadata_sys::init_bucket_metadata_sys(store.clone(), buckets).await;
let object_id = case.name.replace('_', "-");
let bucket = format!("bench-direct-{object_id}");
let key = format!("objects/{object_id}.bin");
store
.make_bucket(
&bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("make bucket");
let payload: Vec<u8> = (0..case.payload_len).map(|idx| (idx % 251) as u8).collect();
let mut reader = ChunkNativePutData::from_vec(payload);
let put_info = store
.put_object(&bucket, &key, &mut reader, &ObjectOptions::default())
.await
.expect("put object");
if case.disk_count > 1 {
assert!(put_info.data_blocks > 1, "expected multi-data-shard object");
}
let (_, expected_len) = case.range.get_offset_length(case.payload_len as i64).expect("range length");
EcstoreBenchEnv {
_temp_dir: temp_dir,
store,
bucket,
key,
range: clone_range_spec(&case.range),
opts: ObjectOptions::default(),
expected_len: expected_len as usize,
expected_copy_mode: case.expected_copy_mode,
}
}
async fn run_ecstore_get_object_chunks_bench(env: &EcstoreBenchEnv) -> (GetObjectChunkCopyMode, usize, usize) {
let mut result = env
.store
.get_object_chunks(&env.bucket, &env.key, Some(clone_range_spec(&env.range)), HeaderMap::new(), &env.opts)
.await
.expect("get object chunks");
let copy_mode = result.copy_mode;
let mut total_len = 0usize;
let mut chunk_count = 0usize;
while let Some(chunk) = result.stream.next().await {
let chunk = chunk.expect("chunk");
total_len += chunk.len();
chunk_count += 1;
}
(copy_mode, total_len, chunk_count)
}
fn build_shard_bytes(case: &BenchCase) -> Vec<Vec<Bytes>> {
let total_len = case.block_size * case.blocks;
let payload: Vec<u8> = (0..total_len).map(|idx| (idx % 251) as u8).collect();
let mut shards = vec![Vec::with_capacity(case.blocks); case.data_shards];
for block in 0..case.blocks {
let block_start = block * case.block_size;
let block_slice = &payload[block_start..block_start + case.block_size];
let shard_width = case.block_size / case.data_shards;
for (shard_idx, shard) in shards.iter_mut().enumerate().take(case.data_shards) {
let shard_start = shard_idx * shard_width;
let shard_end = shard_start + shard_width;
shard.push(Bytes::copy_from_slice(&block_slice[shard_start..shard_end]));
}
}
shards
}
fn build_mapped_streams(shards: &[Vec<Bytes>]) -> Vec<BoxChunkStream> {
shards
.iter()
.map(|shard_chunks| {
let chunks: Vec<_> = shard_chunks
.iter()
.map(|chunk| {
let mapped = MappedChunk::new(chunk.clone(), 0, chunk.len()).expect("mapped chunk");
Ok::<IoChunk, std::io::Error>(IoChunk::Mapped(mapped))
})
.collect();
Box::pin(stream::iter(chunks)) as BoxChunkStream
})
.collect()
}
fn collect_old_assembly(
shards: &[Vec<Bytes>],
data_shards: usize,
block_size: usize,
offset: usize,
length: usize,
) -> Vec<IoChunk> {
if length == 0 {
return Vec::new();
}
let start_block = offset / block_size;
let end_block = offset.saturating_add(length - 1) / block_size;
let mut result = Vec::with_capacity(end_block - start_block + 1);
for block_index in start_block..=end_block {
let block_offset = if block_index == start_block { offset % block_size } else { 0 };
let block_length = if start_block == end_block {
length
} else if block_index == start_block {
block_size - (offset % block_size)
} else if block_index == end_block {
(offset + length) % block_size
} else {
block_size
};
if block_length == 0 {
break;
}
let mut block = BytesMut::with_capacity(block_length);
let mut write_left = block_length;
let mut skip = block_offset;
for shard in shards.iter().take(data_shards) {
let shard_chunk = &shard[block_index];
if skip >= shard_chunk.len() {
skip -= shard_chunk.len();
continue;
}
let available = &shard_chunk[skip..];
skip = 0;
let take = available.len().min(write_left);
block.extend_from_slice(&available[..take]);
write_left -= take;
if write_left == 0 {
break;
}
}
result.push(IoChunk::Shared(block.freeze()));
}
result
}
fn bench_direct_chunk_path(c: &mut Criterion) {
let runtime = Runtime::new().expect("tokio runtime");
let mut group = c.benchmark_group("direct_chunk_path");
for case in bench_cases() {
let shard_bytes = build_shard_bytes(&case);
group.throughput(Throughput::Bytes(case.length as u64));
group.bench_with_input(BenchmarkId::new("slice_forwarding", case.name), &case, |b, case| {
b.iter(|| {
let streams = build_mapped_streams(&shard_bytes);
let chunks = runtime
.block_on(collect_direct_data_shard_chunks_for_benchmark(
streams,
case.data_shards,
case.block_size,
case.blocks * case.block_size,
false,
case.offset,
case.length,
))
.expect("collect direct chunks");
black_box(chunks);
});
});
group.bench_with_input(BenchmarkId::new("assembled_copy", case.name), &case, |b, case| {
b.iter(|| {
let chunks = collect_old_assembly(&shard_bytes, case.data_shards, case.block_size, case.offset, case.length);
black_box(chunks);
});
});
}
group.finish();
}
fn bench_ecstore_get_object_chunks(c: &mut Criterion) {
let runtime = Runtime::new().expect("tokio runtime");
let mut group = c.benchmark_group("ecstore_get_object_chunks");
group.sample_size(10);
for case in ecstore_bench_cases() {
let env = runtime.block_on(build_ecstore_bench_env(&case));
let (copy_mode, total_len, _) = runtime.block_on(run_ecstore_get_object_chunks_bench(&env));
assert_eq!(copy_mode, env.expected_copy_mode);
assert_eq!(total_len, env.expected_len);
group.throughput(Throughput::Bytes(env.expected_len as u64));
group.bench_with_input(BenchmarkId::new("drain", case.name), &env, |b, env| {
b.iter(|| {
let result = runtime.block_on(run_ecstore_get_object_chunks_bench(env));
black_box(result);
});
});
}
group.finish();
}
criterion_group!(benches, bench_direct_chunk_path, bench_ecstore_get_object_chunks);
criterion_main!(benches);
@@ -0,0 +1,393 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use futures_util::StreamExt;
use http::HeaderMap;
use rustfs_ecstore::bucket::metadata_sys;
use rustfs_ecstore::disk::endpoint::Endpoint;
use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use rustfs_ecstore::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
use rustfs_ecstore::store::{ECStore, init_local_disks};
use rustfs_ecstore::store_api::{
BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, GetObjectChunkCopyMode, HTTPRangeSpec, MakeBucketOptions,
MultipartOperations, ObjectOperations, ObjectOptions,
};
use std::hint::black_box;
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicU16, Ordering};
use tempfile::TempDir;
use tokio::runtime::Runtime;
use tokio_util::sync::CancellationToken;
#[derive(Clone)]
enum ReconstructedReadSpec {
PartNumber {
part_number: usize,
missing_part_name: &'static str,
},
Range {
start: u64,
end: u64,
missing_part_name: &'static str,
},
}
#[derive(Clone)]
struct ReconstructedBenchCase {
object_id: &'static str,
name: &'static str,
read_spec: ReconstructedReadSpec,
}
struct ReconstructedBenchEnv {
store: Arc<ECStore>,
bucket: String,
key: String,
range: HTTPRangeSpec,
opts: ObjectOptions,
expected_len: usize,
}
struct ReconstructedBenchSuite {
_temp_dir: TempDir,
store: Arc<ECStore>,
disk_paths: Vec<PathBuf>,
}
const MULTIPART_PART_ONE_LEN: usize = 5 * 1024 * 1024;
const MULTIPART_PART_TWO_LEN: usize = 5 * 1024 * 1024 + 137;
const MULTIPART_PART_THREE_LEN: usize = 1024 * 1024 + 77;
fn reconstructed_bench_cases() -> [ReconstructedBenchCase; 4] {
[
ReconstructedBenchCase {
object_id: "mp-part2",
name: "multi_disk_missing_shard_multipart_part2",
read_spec: ReconstructedReadSpec::PartNumber {
part_number: 2,
missing_part_name: "part.2",
},
},
ReconstructedBenchCase {
object_id: "mp-cross-range",
name: "multi_disk_missing_shard_multipart_cross_part_range",
read_spec: ReconstructedReadSpec::Range {
start: (MULTIPART_PART_ONE_LEN - 32 * 1024) as u64,
end: (MULTIPART_PART_ONE_LEN + 96 * 1024) as u64,
missing_part_name: "part.2",
},
},
ReconstructedBenchCase {
object_id: "mp-part3",
name: "multi_disk_missing_shard_multipart_part3",
read_spec: ReconstructedReadSpec::PartNumber {
part_number: 3,
missing_part_name: "part.3",
},
},
ReconstructedBenchCase {
object_id: "mp-cross-final-range",
name: "multi_disk_missing_shard_multipart_cross_final_part_range",
read_spec: ReconstructedReadSpec::Range {
start: (MULTIPART_PART_ONE_LEN + MULTIPART_PART_TWO_LEN - 32 * 1024) as u64,
end: (MULTIPART_PART_ONE_LEN + MULTIPART_PART_TWO_LEN + 96 * 1024) as u64,
missing_part_name: "part.3",
},
},
]
}
fn next_loopback_addr() -> SocketAddr {
static NEXT_PORT: AtomicU16 = AtomicU16::new(39113);
let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed);
SocketAddr::from(([127, 0, 0, 1], port))
}
fn clone_range_spec(range: &HTTPRangeSpec) -> HTTPRangeSpec {
HTTPRangeSpec {
is_suffix_length: range.is_suffix_length,
start: range.start,
end: range.end,
}
}
fn build_endpoint_pools(paths: &[PathBuf]) -> EndpointServerPools {
let mut endpoints = Vec::with_capacity(paths.len());
for (idx, disk_path) in paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf8 path")).expect("endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(idx);
endpoints.push(endpoint);
}
EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: paths.len(),
endpoints: Endpoints::from(endpoints),
cmd_line: "bench".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
}])
}
fn find_part_files(root: &Path, part_name: &str, out: &mut Vec<PathBuf>) {
let Ok(entries) = std::fs::read_dir(root) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
find_part_files(&path, part_name, out);
continue;
}
if path.file_name().and_then(|name| name.to_str()) == Some(part_name) {
out.push(path);
}
}
}
async fn remove_part_files(root: &Path, part_name: &str) -> Vec<(PathBuf, Vec<u8>)> {
let mut paths = Vec::new();
find_part_files(root, part_name, &mut paths);
let mut removed = Vec::with_capacity(paths.len());
for path in paths {
let content = tokio::fs::read(&path).await.expect("read part file before removal");
tokio::fs::remove_file(&path).await.expect("remove part file");
removed.push((path, content));
}
removed
}
async fn restore_part_files(files: Vec<(PathBuf, Vec<u8>)>) {
for (path, content) in files {
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent)
.await
.expect("ensure parent for part restore");
}
tokio::fs::write(&path, content).await.expect("restore part file");
}
}
async fn run_reconstructed_get_object_chunks(
store: &Arc<ECStore>,
bucket: &str,
key: &str,
range: &HTTPRangeSpec,
opts: &ObjectOptions,
) -> (GetObjectChunkCopyMode, usize, usize) {
let mut result = store
.get_object_chunks(bucket, key, Some(clone_range_spec(range)), HeaderMap::new(), opts)
.await
.expect("get object chunks");
let copy_mode = result.copy_mode;
let mut total_len = 0usize;
let mut chunk_count = 0usize;
while let Some(chunk) = result.stream.next().await {
let chunk = chunk.expect("chunk");
total_len += chunk.len();
chunk_count += 1;
}
(copy_mode, total_len, chunk_count)
}
async fn create_multipart_object(store: &Arc<ECStore>, bucket: &str, key: &str, parts: &[Vec<u8>]) -> usize {
let upload = store
.new_multipart_upload(bucket, key, &ObjectOptions::default())
.await
.expect("new multipart upload");
let mut completed_parts = Vec::with_capacity(parts.len());
for (idx, part) in parts.iter().enumerate() {
let mut reader = ChunkNativePutData::from_vec(part.clone());
let part_info = store
.put_object_part(bucket, key, &upload.upload_id, idx + 1, &mut reader, &ObjectOptions::default())
.await
.expect("put object part");
completed_parts.push(CompletePart {
part_num: idx + 1,
etag: part_info.etag,
..Default::default()
});
}
store
.clone()
.complete_multipart_upload(bucket, key, &upload.upload_id, completed_parts, &ObjectOptions::default())
.await
.expect("complete multipart upload");
parts.iter().map(Vec::len).sum()
}
async fn build_reconstructed_bench_suite() -> ReconstructedBenchSuite {
let temp_dir = tempfile::tempdir().expect("tempdir");
let disk_paths: Vec<_> = (1..=4).map(|idx| temp_dir.path().join(format!("disk{idx}"))).collect();
for disk_path in &disk_paths {
tokio::fs::create_dir_all(disk_path).await.expect("create disk dir");
}
let endpoint_pools = build_endpoint_pools(&disk_paths);
GLOBAL_LOCAL_DISK_MAP.write().await.clear();
GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
let store = ECStore::new(next_loopback_addr(), endpoint_pools, CancellationToken::new())
.await
.expect("create ecstore");
let buckets = store
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("list buckets")
.into_iter()
.map(|bucket| bucket.name)
.collect();
metadata_sys::init_bucket_metadata_sys(store.clone(), buckets).await;
ReconstructedBenchSuite {
_temp_dir: temp_dir,
store,
disk_paths,
}
}
async fn build_reconstructed_bench_env(suite: &ReconstructedBenchSuite, case: &ReconstructedBenchCase) -> ReconstructedBenchEnv {
let bucket = format!("bench-r-{}", case.object_id);
let key = format!("objects/{}.bin", case.object_id);
suite
.store
.make_bucket(
&bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("make bucket");
let part_one: Vec<u8> = (0..MULTIPART_PART_ONE_LEN).map(|idx| (idx % 251) as u8).collect();
let part_two: Vec<u8> = (0..MULTIPART_PART_TWO_LEN).map(|idx| ((idx + 11) % 251) as u8).collect();
let part_three: Vec<u8> = (0..MULTIPART_PART_THREE_LEN).map(|idx| ((idx + 29) % 251) as u8).collect();
let payload_len = create_multipart_object(&suite.store, &bucket, &key, &[part_one, part_two, part_three]).await;
let info = suite
.store
.get_object_info(&bucket, &key, &ObjectOptions::default())
.await
.expect("get object info");
let (range, opts, missing_part_name) = match case.read_spec.clone() {
ReconstructedReadSpec::PartNumber {
part_number,
missing_part_name,
} => {
let range = HTTPRangeSpec::from_object_info(&info, part_number).expect("part_number range");
let opts = ObjectOptions {
part_number: Some(part_number),
..Default::default()
};
(range, opts, missing_part_name)
}
ReconstructedReadSpec::Range {
start,
end,
missing_part_name,
} => (
HTTPRangeSpec {
is_suffix_length: false,
start: start as i64,
end: end as i64,
},
ObjectOptions::default(),
missing_part_name,
),
};
let (_, expected_len) = range.get_offset_length(payload_len as i64).expect("range length");
let mut selected_reconstructed = false;
for disk_path in &suite.disk_paths {
let object_root = disk_path
.join(&bucket)
.join("objects")
.join(format!("{}.bin", case.object_id));
let removed_parts = remove_part_files(&object_root, missing_part_name).await;
if removed_parts.is_empty() {
continue;
}
let (copy_mode, total_len, _) = run_reconstructed_get_object_chunks(&suite.store, &bucket, &key, &range, &opts).await;
if copy_mode == GetObjectChunkCopyMode::Reconstructed && total_len == expected_len as usize {
selected_reconstructed = true;
break;
}
restore_part_files(removed_parts).await;
}
assert!(
selected_reconstructed,
"failed to select a missing shard placement that triggers reconstructed copy mode for benchmark case {}",
case.name
);
ReconstructedBenchEnv {
store: suite.store.clone(),
bucket,
key,
range: clone_range_spec(&range),
opts,
expected_len: expected_len as usize,
}
}
async fn run_reconstructed_get_object_chunks_bench(env: &ReconstructedBenchEnv) -> (GetObjectChunkCopyMode, usize, usize) {
run_reconstructed_get_object_chunks(&env.store, &env.bucket, &env.key, &env.range, &env.opts).await
}
fn bench_reconstructed_chunk_path(c: &mut Criterion) {
let runtime = Runtime::new().expect("tokio runtime");
let suite = runtime.block_on(build_reconstructed_bench_suite());
let mut group = c.benchmark_group("reconstructed_chunk_path");
group.sample_size(10);
for case in reconstructed_bench_cases() {
let env = runtime.block_on(build_reconstructed_bench_env(&suite, &case));
let (copy_mode, total_len, _) = runtime.block_on(run_reconstructed_get_object_chunks_bench(&env));
assert_eq!(copy_mode, GetObjectChunkCopyMode::Reconstructed);
assert_eq!(total_len, env.expected_len);
group.throughput(Throughput::Bytes(env.expected_len as u64));
group.bench_with_input(BenchmarkId::new("drain", case.name), &env, |b, env| {
b.iter(|| {
let result = runtime.block_on(run_reconstructed_get_object_chunks_bench(env));
black_box(result);
});
});
}
group.finish();
}
criterion_group!(benches, bench_reconstructed_chunk_path);
criterion_main!(benches);
+18 -1
View File
@@ -119,6 +119,16 @@ run_large_data_test() {
print_success "Large-dataset tests completed"
}
# Run direct chunk path benchmarks
run_direct_chunk_benchmark() {
print_info "📦 Starting direct chunk path benchmarks..."
echo "================================================"
cargo bench --bench direct_chunk_benchmark
print_success "Direct chunk path benchmarks completed"
}
# Generate comparison report
generate_comparison_report() {
print_info "📊 Generating performance report..."
@@ -168,6 +178,7 @@ show_help() {
echo " full Run the full benchmark suite"
echo " performance Run detailed performance tests"
echo " simd Run the SIMD-only tests"
echo " direct Run the direct chunk path benchmarks"
echo " large Run large-dataset tests"
echo " clean Remove previous results"
echo " help Show this help message"
@@ -177,6 +188,7 @@ show_help() {
echo " $0 performance # Detailed performance test"
echo " $0 full # Full benchmark suite"
echo " $0 simd # SIMD-only benchmark"
echo " $0 direct # Direct chunk path benchmark"
echo " $0 large # Large-dataset benchmark"
echo ""
echo "Features:"
@@ -240,6 +252,11 @@ main() {
run_simd_benchmark
generate_comparison_report
;;
"direct")
cleanup
run_direct_chunk_benchmark
generate_comparison_report
;;
"large")
cleanup
run_large_data_test
@@ -263,4 +280,4 @@ main() {
}
# Launch script
main "$@"
main "$@"
+780 -10
View File
@@ -14,13 +14,328 @@
use crate::disk::{self, DiskAPI as _, DiskStore, error::DiskError};
use crate::erasure_coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
use bytes::Bytes;
use crate::store_api::{GetObjectChunkCopyMode, GetObjectChunkPath, GetObjectChunkResult};
use bytes::{Bytes, BytesMut};
use futures_util::{StreamExt, stream};
use rustfs_io_core::{BoxChunkStream, IoChunk};
use rustfs_utils::HashAlgorithm;
use std::collections::VecDeque;
use std::io::Cursor;
use std::time::Instant;
use tokio::io::AsyncRead;
use tracing::debug;
const BITROT_READ_OPERATION: &str = "bitrot_read";
fn classify_chunk_copy_mode(source_direct: bool, copied: bool) -> GetObjectChunkCopyMode {
if copied {
GetObjectChunkCopyMode::SingleCopy
} else if source_direct {
GetObjectChunkCopyMode::TrueZeroCopy
} else {
GetObjectChunkCopyMode::SharedBytes
}
}
struct ChunkSpan {
bytes: Bytes,
chunk: IoChunk,
copied: bool,
}
fn take_contiguous_chunk_span(chunk: &IoChunk, offset: usize, len: usize) -> std::io::Result<ChunkSpan> {
match chunk {
IoChunk::Shared(bytes) => {
let bytes = bytes.slice(offset..offset + len);
Ok(ChunkSpan {
bytes: bytes.clone(),
chunk: IoChunk::Shared(bytes),
copied: false,
})
}
IoChunk::Mapped(mapped) => {
let chunk = IoChunk::Mapped(mapped.slice(offset, len)?);
let bytes = chunk.as_bytes();
Ok(ChunkSpan {
bytes,
chunk,
copied: false,
})
}
IoChunk::Pooled(pooled) => {
let chunk = IoChunk::Pooled(pooled.slice(offset, len)?);
let bytes = chunk.as_bytes();
Ok(ChunkSpan {
bytes,
chunk,
copied: false,
})
}
}
}
struct BitrotChunkSource {
source_stream: BoxChunkStream,
source_chunks: VecDeque<IoChunk>,
source_chunk_offset: usize,
source_buffered_bytes: usize,
source_done: bool,
}
struct BitrotChunkStreamState {
source: BitrotChunkSource,
decoded_remaining: usize,
trim_prefix: usize,
output_remaining: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify: bool,
}
struct ChunkCursor<'a> {
chunks: &'a [IoChunk],
chunk_index: usize,
chunk_offset: usize,
consumed: usize,
total_len: usize,
}
impl<'a> ChunkCursor<'a> {
fn new(chunks: &'a [IoChunk]) -> Self {
Self {
chunks,
chunk_index: 0,
chunk_offset: 0,
consumed: 0,
total_len: chunks.iter().map(IoChunk::len).sum(),
}
}
fn remaining(&self) -> usize {
self.total_len.saturating_sub(self.consumed)
}
fn skip_empty_chunks(&mut self) {
while let Some(chunk) = self.chunks.get(self.chunk_index) {
if self.chunk_offset < chunk.len() {
break;
}
self.chunk_index += 1;
self.chunk_offset = 0;
}
}
fn advance(&mut self, len: usize) {
self.consumed += len;
self.chunk_offset += len;
self.skip_empty_chunks();
}
fn take_span(&mut self, len: usize) -> std::io::Result<ChunkSpan> {
self.skip_empty_chunks();
if self.remaining() < len {
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
}
let Some(chunk) = self.chunks.get(self.chunk_index) else {
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "missing bitrot chunk source"));
};
let available = chunk.len().saturating_sub(self.chunk_offset);
if len <= available {
let span = take_contiguous_chunk_span(chunk, self.chunk_offset, len)?;
self.advance(len);
return Ok(span);
}
let mut aggregate = BytesMut::with_capacity(len);
let mut remaining = len;
while remaining > 0 {
self.skip_empty_chunks();
let Some(chunk) = self.chunks.get(self.chunk_index) else {
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
};
let available = chunk.len().saturating_sub(self.chunk_offset);
let take = available.min(remaining);
aggregate.extend_from_slice(&chunk.as_bytes()[self.chunk_offset..self.chunk_offset + take]);
self.advance(take);
remaining -= take;
}
let bytes = aggregate.freeze();
Ok(ChunkSpan {
bytes: bytes.clone(),
chunk: IoChunk::Shared(bytes),
copied: true,
})
}
}
impl BitrotChunkSource {
fn new(source_stream: BoxChunkStream, source_chunks: VecDeque<IoChunk>, source_done: bool) -> Self {
let source_buffered_bytes = source_chunks.iter().map(IoChunk::len).sum();
Self {
source_stream,
source_chunks,
source_chunk_offset: 0,
source_buffered_bytes,
source_done,
}
}
fn skip_empty_chunks(&mut self) {
while let Some(chunk) = self.source_chunks.front() {
if self.source_chunk_offset < chunk.len() {
break;
}
self.source_chunks.pop_front();
self.source_chunk_offset = 0;
}
}
async fn fill(&mut self, min_bytes: usize) -> std::io::Result<()> {
while self.source_buffered_bytes < min_bytes && !self.source_done {
match self.source_stream.next().await {
Some(Ok(chunk)) => {
self.source_buffered_bytes += chunk.len();
self.source_chunks.push_back(chunk);
}
Some(Err(err)) => return Err(err),
None => self.source_done = true,
}
}
if self.source_buffered_bytes < min_bytes {
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
}
Ok(())
}
fn advance(&mut self, len: usize) {
self.source_buffered_bytes = self.source_buffered_bytes.saturating_sub(len);
self.source_chunk_offset += len;
self.skip_empty_chunks();
}
fn take_span(&mut self, len: usize) -> std::io::Result<ChunkSpan> {
self.skip_empty_chunks();
if self.source_buffered_bytes < len {
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
}
let Some(chunk) = self.source_chunks.front() else {
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "missing bitrot chunk source"));
};
let available = chunk.len().saturating_sub(self.source_chunk_offset);
if len <= available {
let span = take_contiguous_chunk_span(chunk, self.source_chunk_offset, len)?;
self.advance(len);
return Ok(span);
}
let mut aggregate = BytesMut::with_capacity(len);
let mut remaining = len;
while remaining > 0 {
self.skip_empty_chunks();
let Some(chunk) = self.source_chunks.front() else {
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
};
let available = chunk.len().saturating_sub(self.source_chunk_offset);
let take = available.min(remaining);
aggregate.extend_from_slice(&chunk.as_bytes()[self.source_chunk_offset..self.source_chunk_offset + take]);
self.advance(take);
remaining -= take;
}
let bytes = aggregate.freeze();
Ok(ChunkSpan {
bytes: bytes.clone(),
chunk: IoChunk::Shared(bytes),
copied: true,
})
}
}
impl BitrotChunkStreamState {
#[allow(clippy::too_many_arguments)]
fn new(
source_stream: BoxChunkStream,
source_chunks: VecDeque<IoChunk>,
source_done: bool,
decoded_remaining: usize,
trim_prefix: usize,
output_remaining: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify: bool,
) -> Self {
Self {
source: BitrotChunkSource::new(source_stream, source_chunks, source_done),
decoded_remaining,
trim_prefix,
output_remaining,
shard_size,
checksum_algo,
skip_verify,
}
}
fn hash_size(&self) -> usize {
self.checksum_algo.size()
}
async fn next_verified_chunk(&mut self) -> std::io::Result<Option<IoChunk>> {
let hash_size = self.hash_size();
while self.output_remaining > 0 && self.decoded_remaining > 0 {
let data_len = self.shard_size.min(self.decoded_remaining);
let expected_hash = if hash_size > 0 {
self.source.fill(hash_size).await?;
Some(self.source.take_span(hash_size)?)
} else {
None
};
self.source.fill(data_len).await?;
let data_span = self.source.take_span(data_len)?;
if let Some(expected_hash) = expected_hash
&& !self.skip_verify
&& self.checksum_algo.hash_encode(data_span.bytes.as_ref()).as_ref() != expected_hash.bytes.as_ref()
{
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
}
self.decoded_remaining -= data_len;
if self.trim_prefix >= data_len {
self.trim_prefix -= data_len;
continue;
}
let start = self.trim_prefix;
self.trim_prefix = 0;
let take = (data_len - start).min(self.output_remaining);
self.output_remaining -= take;
let chunk = if start == 0 && take == data_len {
data_span.chunk
} else {
data_span.chunk.slice(start, take)?
};
if !chunk.is_empty() {
return Ok(Some(chunk));
}
}
Ok(None)
}
}
/// Create a BitrotReader from either inline data or disk file stream
///
/// # Parameters
@@ -65,20 +380,39 @@ pub async fn create_bitrot_reader(
} else if let Some(disk) = disk {
// Read from disk
if use_zero_copy {
if !disk.is_local() {
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
rustfs_io_metrics::record_io_fallback(
rustfs_io_metrics::IoStage::ReadSetup,
rustfs_io_metrics::FallbackReason::NonLocalBackend,
);
let rd = disk.read_file_stream(bucket, path, offset, length).await?;
let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
return Ok(Some(reader));
}
// Try zero-copy read first (uses mmap on Unix)
let start = Instant::now();
match disk.read_file_zero_copy(bucket, path, offset, length).await {
Ok(bytes) => {
let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
// Record zero-copy metrics
rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms);
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Fast);
// `read_file_zero_copy()` returns a shared `Bytes` view, but it may still
// internally aggregate multiple chunk windows. The exact chunk-native copy
// mode is only preserved by `create_bitrot_chunk_stream()`.
rustfs_io_metrics::record_io_copy_mode(
BITROT_READ_OPERATION,
rustfs_io_metrics::CopyMode::SharedBytes,
bytes.len(),
);
// Log successful zero-copy read
debug!(
size = bytes.len(),
duration_ms,
path = %path,
"zero_copy_read_success"
"bitrot_fast_read_success"
);
// Wrap Bytes in Cursor for AsyncRead
@@ -93,14 +427,16 @@ pub async fn create_bitrot_reader(
Ok(Some(reader))
}
Err(e) => {
// Record zero-copy fallback
rustfs_io_metrics::record_zero_copy_fallback(&format!("{:?}", e));
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
rustfs_io_metrics::record_io_fallback(
rustfs_io_metrics::IoStage::ReadSetup,
rustfs_io_metrics::FallbackReason::Unknown,
);
// Log zero-copy fallback
debug!(
reason = %format!("{:?}", e),
reason = %e,
path = %path,
"zero_copy_fallback"
"bitrot_fast_read_fallback"
);
// Fall back to regular stream read on error
@@ -117,6 +453,7 @@ pub async fn create_bitrot_reader(
}
}
} else {
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
// Use regular stream read
match disk.read_file_stream(bucket, path, offset, length).await {
Ok(rd) => {
@@ -132,6 +469,198 @@ pub async fn create_bitrot_reader(
}
}
/// Create a chunk stream from bitrot-encoded data, preserving source chunk provenance when possible.
#[allow(clippy::too_many_arguments)]
pub async fn create_bitrot_chunk_stream(
inline_data: Option<&[u8]>,
disk: Option<&DiskStore>,
bucket: &str,
path: &str,
offset: usize,
length: usize,
total_data_size: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify: bool,
use_zero_copy: bool,
) -> disk::error::Result<Option<GetObjectChunkResult>> {
let fetch_start = (offset / shard_size) * shard_size;
let fetch_end = (offset + length).div_ceil(shard_size) * shard_size;
let fetch_end = fetch_end.min(total_data_size);
let fetch_length = fetch_end.saturating_sub(fetch_start);
let trim_prefix = offset.saturating_sub(fetch_start);
let hash_size = checksum_algo.size();
let encoded_length = fetch_length.div_ceil(shard_size) * hash_size + fetch_length;
let encoded_offset = fetch_start.div_ceil(shard_size) * hash_size + fetch_start;
let mut source_done = false;
let (source_stream, mut prefetched_chunks, source_direct) = if let Some(data) = inline_data {
source_done = true;
let mut chunks = VecDeque::new();
chunks.push_back(IoChunk::Shared(
Bytes::copy_from_slice(data).slice(encoded_offset..encoded_offset + encoded_length),
));
let source_stream: BoxChunkStream = Box::pin(stream::empty::<std::io::Result<IoChunk>>());
(source_stream, chunks, false)
} else if let Some(disk) = disk {
if use_zero_copy {
let mut source_stream = disk.read_file_chunks(bucket, path, encoded_offset, encoded_length).await?;
let mut prefetched_chunks = VecDeque::new();
let mut direct = true;
while prefetched_chunks.len() < 2 {
let Some(chunk) = source_stream.next().await else {
source_done = true;
break;
};
let chunk = chunk?;
direct &= matches!(chunk, IoChunk::Mapped(_));
prefetched_chunks.push_back(chunk);
}
(source_stream, prefetched_chunks, direct)
} else {
source_done = true;
let bytes = disk.read_file_zero_copy(bucket, path, encoded_offset, encoded_length).await?;
let mut chunks = VecDeque::new();
chunks.push_back(IoChunk::Shared(bytes));
let source_stream: BoxChunkStream = Box::pin(stream::empty::<std::io::Result<IoChunk>>());
(source_stream, chunks, false)
}
} else {
return Ok(None);
};
let copied = predicted_stream_copy(encoded_length, shard_size, checksum_algo.size(), &prefetched_chunks, source_done);
let state = BitrotChunkStreamState::new(
source_stream,
std::mem::take(&mut prefetched_chunks),
source_done,
fetch_length,
trim_prefix,
length,
shard_size,
checksum_algo,
skip_verify,
);
let stream = stream::unfold(Some(state), |state| async move {
let mut state = match state {
Some(state) => state,
None => return None,
};
match state.next_verified_chunk().await {
Ok(Some(chunk)) => {
let next_state = if state.output_remaining == 0 { None } else { Some(state) };
Some((Ok::<IoChunk, std::io::Error>(chunk), next_state))
}
Ok(None) => None,
Err(err) => Some((Err(err), None)),
}
});
Ok(Some(GetObjectChunkResult {
stream: Box::pin(stream),
path: GetObjectChunkPath::Direct,
copy_mode: classify_chunk_copy_mode(source_direct, copied),
}))
}
fn predicted_stream_copy(
encoded_length: usize,
shard_size: usize,
hash_size: usize,
prefetched_chunks: &VecDeque<IoChunk>,
source_done: bool,
) -> bool {
if prefetched_chunks.is_empty() {
return false;
}
if source_done && prefetched_chunks.len() == 1 {
return false;
}
let full_frame_len = hash_size + shard_size;
if full_frame_len == 0 {
return false;
}
let first_window_len = prefetched_chunks.front().map(IoChunk::len).unwrap_or(encoded_length);
encoded_length > first_window_len && !first_window_len.is_multiple_of(full_frame_len)
}
fn trim_chunk_vec(chunks: Vec<IoChunk>, offset: usize, length: usize) -> std::io::Result<Vec<IoChunk>> {
let mut skip = offset;
let mut remaining = length;
let mut result = Vec::new();
for chunk in chunks {
if remaining == 0 {
break;
}
let chunk_len = chunk.len();
if skip >= chunk_len {
skip -= chunk_len;
continue;
}
let start = skip;
let take = (chunk_len - start).min(remaining);
result.push(chunk.slice(start, take)?);
remaining -= take;
skip = 0;
}
Ok(result)
}
fn decode_bitrot_chunk_source(
source_chunks: &[IoChunk],
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify: bool,
) -> std::io::Result<(Vec<IoChunk>, bool)> {
let hash_size = checksum_algo.size();
let mut cursor = ChunkCursor::new(source_chunks);
let mut result = Vec::new();
let mut copied = false;
while cursor.remaining() > 0 {
let expected_hash = if hash_size > 0 {
Some(cursor.take_span(hash_size)?)
} else {
None
};
let data_len = shard_size.min(cursor.remaining());
if data_len == 0 {
break;
}
let data_span = cursor.take_span(data_len)?;
copied |= data_span.copied;
if let Some(expected_hash) = expected_hash {
copied |= expected_hash.copied;
if !skip_verify && checksum_algo.hash_encode(data_span.bytes.as_ref()).as_ref() != expected_hash.bytes.as_ref() {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
}
}
result.push(data_span.chunk);
}
Ok((result, copied))
}
#[doc(hidden)]
pub fn decode_bitrot_chunk_source_for_bench(
source_chunks: &[IoChunk],
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify: bool,
) -> std::io::Result<(Vec<IoChunk>, bool)> {
decode_bitrot_chunk_source(source_chunks, shard_size, checksum_algo, skip_verify)
}
/// Create a new BitrotWriterWrapper based on the provided parameters
///
/// # Parameters
@@ -176,6 +705,7 @@ pub async fn create_bitrot_writer(
#[cfg(test)]
mod tests {
use super::*;
use futures_util::StreamExt;
#[tokio::test]
async fn test_create_bitrot_reader_with_inline_data() {
@@ -226,6 +756,246 @@ mod tests {
assert!(result.unwrap().is_some());
}
#[tokio::test]
async fn test_create_bitrot_chunk_stream_with_inline_data() {
let shard_size = 4;
let checksum_algo = HashAlgorithm::HighwayHash256S;
let shard1 = b"abcd";
let shard2 = b"ef";
let mut encoded = Vec::new();
encoded.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
encoded.extend_from_slice(shard1);
encoded.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
encoded.extend_from_slice(shard2);
let mut stream = create_bitrot_chunk_stream(
Some(&encoded),
None,
"test-bucket",
"test-path",
0,
shard1.len() + shard2.len(),
shard1.len() + shard2.len(),
shard_size,
checksum_algo,
false,
false,
)
.await
.unwrap()
.unwrap()
.stream;
let mut collected = Vec::new();
while let Some(chunk) = stream.next().await {
collected.extend_from_slice(&chunk.unwrap().as_bytes());
}
assert_eq!(collected, b"abcdef");
}
#[tokio::test]
async fn test_create_bitrot_chunk_stream_detects_hash_mismatch() {
let shard_size = 4;
let checksum_algo = HashAlgorithm::HighwayHash256S;
let shard = b"abcd";
let mut encoded = Vec::new();
let mut bad_hash = checksum_algo.hash_encode(shard).as_ref().to_vec();
bad_hash[0] ^= 0xFF;
encoded.extend_from_slice(&bad_hash);
encoded.extend_from_slice(shard);
let result = create_bitrot_chunk_stream(
Some(&encoded),
None,
"test-bucket",
"test-path",
0,
shard.len(),
shard.len(),
shard_size,
checksum_algo,
false,
false,
)
.await;
let mut stream = result.unwrap().unwrap().stream;
let err = stream.next().await.unwrap().unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(err.to_string().contains("bitrot hash mismatch"));
}
#[tokio::test]
async fn test_create_bitrot_chunk_stream_trims_range_after_decode() {
let shard_size = 4;
let checksum_algo = HashAlgorithm::HighwayHash256S;
let shard1 = b"abcd";
let shard2 = b"efgh";
let mut encoded = Vec::new();
encoded.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
encoded.extend_from_slice(shard1);
encoded.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
encoded.extend_from_slice(shard2);
let mut stream = create_bitrot_chunk_stream(
Some(&encoded),
None,
"test-bucket",
"test-path",
1,
5,
shard1.len() + shard2.len(),
shard_size,
checksum_algo,
false,
false,
)
.await
.unwrap()
.unwrap()
.stream;
let mut collected = Vec::new();
while let Some(chunk) = stream.next().await {
collected.extend_from_slice(&chunk.unwrap().as_bytes());
}
assert_eq!(collected, b"bcdef");
}
#[test]
fn test_decode_bitrot_chunk_source_preserves_aligned_multi_chunk_slices() {
let shard_size = 4;
let checksum_algo = HashAlgorithm::Md5;
let shard1 = b"abcd";
let shard2 = b"efgh";
let mut encoded_chunk_one = Vec::new();
encoded_chunk_one.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
encoded_chunk_one.extend_from_slice(shard1);
let mut encoded_chunk_two = Vec::new();
encoded_chunk_two.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
encoded_chunk_two.extend_from_slice(shard2);
let source_chunks = vec![
IoChunk::Shared(Bytes::from(encoded_chunk_one)),
IoChunk::Shared(Bytes::from(encoded_chunk_two)),
];
let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
assert!(!copied, "frame-aligned multi-chunk source should not require aggregate copies");
assert_eq!(decoded.len(), 2);
assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
assert_eq!(decoded[1].as_bytes(), Bytes::from_static(b"efgh"));
}
#[test]
fn test_decode_bitrot_chunk_source_marks_cross_chunk_frame_as_copied() {
let shard_size = 4;
let checksum_algo = HashAlgorithm::Md5;
let shard1 = b"abcd";
let shard2 = b"efgh";
let hash1 = checksum_algo.hash_encode(shard1).as_ref().to_vec();
let hash2 = checksum_algo.hash_encode(shard2).as_ref().to_vec();
let mut encoded = Vec::new();
encoded.extend_from_slice(&hash1);
encoded.extend_from_slice(shard1);
encoded.extend_from_slice(&hash2);
encoded.extend_from_slice(shard2);
let split = hash1.len() + 2;
let source_chunks = vec![
IoChunk::Shared(Bytes::copy_from_slice(&encoded[..split])),
IoChunk::Shared(Bytes::copy_from_slice(&encoded[split..])),
];
let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
assert!(copied, "cross-chunk frame should be classified as requiring a copy");
assert_eq!(decoded.len(), 2);
assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
assert_eq!(decoded[1].as_bytes(), Bytes::from_static(b"efgh"));
}
#[test]
fn test_decode_bitrot_chunk_source_preserves_pooled_single_chunk_slice() {
let shard_size = 4;
let checksum_algo = HashAlgorithm::Md5;
let shard = b"abcd";
let mut encoded = Vec::new();
encoded.extend_from_slice(checksum_algo.hash_encode(shard).as_ref());
encoded.extend_from_slice(shard);
let source_chunks = vec![IoChunk::Pooled(rustfs_io_core::PooledChunk::from_vec(encoded))];
let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
assert!(!copied, "single pooled chunk slice should preserve provenance without copy");
assert_eq!(decoded.len(), 1);
assert!(matches!(&decoded[0], IoChunk::Pooled(_)));
assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
}
#[tokio::test]
async fn test_bitrot_chunk_source_marks_cross_chunk_take_as_copied() {
let source_stream: BoxChunkStream = Box::pin(stream::iter(vec![
Ok(IoChunk::Shared(Bytes::from_static(b"ab"))),
Ok(IoChunk::Shared(Bytes::from_static(b"cd"))),
]));
let mut source = BitrotChunkSource::new(source_stream, VecDeque::new(), false);
source.fill(4).await.expect("source fill should succeed");
let span = source.take_span(4).expect("cross-chunk take should succeed");
assert!(span.copied, "cross-chunk take should be classified as copied");
assert_eq!(span.bytes, Bytes::from_static(b"abcd"));
assert_eq!(span.chunk.as_bytes(), Bytes::from_static(b"abcd"));
}
#[tokio::test]
async fn test_bitrot_chunk_stream_state_yields_verified_prefix_before_later_truncation() {
let shard_size = 4;
let checksum_algo = HashAlgorithm::Md5;
let shard1 = b"abcd";
let shard2 = b"efgh";
let mut first_frame = Vec::new();
first_frame.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
first_frame.extend_from_slice(shard1);
let mut second_frame_prefix = Vec::new();
second_frame_prefix.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
second_frame_prefix.extend_from_slice(&shard2[..2]);
let source_stream: BoxChunkStream = Box::pin(stream::iter(vec![
Ok(IoChunk::Shared(Bytes::from(first_frame))),
Ok(IoChunk::Shared(Bytes::from(second_frame_prefix))),
]));
let mut state = BitrotChunkStreamState::new(
source_stream,
VecDeque::new(),
false,
shard1.len() + shard2.len(),
0,
shard1.len() + shard2.len(),
shard_size,
checksum_algo,
false,
);
let first = state.next_verified_chunk().await.unwrap().unwrap();
assert_eq!(first.as_bytes(), Bytes::from_static(b"abcd"));
let err = state.next_verified_chunk().await.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
assert!(err.to_string().contains("truncated bitrot chunk source"));
}
#[tokio::test]
async fn test_create_bitrot_reader_with_inline_offset_starts_at_requested_shard() {
let shard_size = 4;
+5 -9
View File
@@ -17,7 +17,7 @@
use crate::bucket::metadata::BUCKET_METADATA_FILE;
use crate::bucket::replication::{decode_resync_file, encode_resync_file};
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::store_api::{BucketOptions, ObjectOptions, PutObjReader, StorageAPI};
use crate::store_api::{BucketOptions, ChunkNativePutData, ObjectOptions, StorageAPI};
use http::HeaderMap;
use rustfs_policy::auth::UserIdentity;
use rustfs_policy::policy::PolicyDoc;
@@ -263,10 +263,8 @@ async fn migrate_one_if_missing<S: StorageAPI>(
}
};
if let Err(e) = store
.put_object(RUSTFS_META_BUCKET, path, &mut PutObjReader::from_vec(data), opts)
.await
{
let mut put_data = ChunkNativePutData::from_vec(data);
if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await {
warn!("write {label}: {e}");
} else {
info!("Migrated {label}");
@@ -343,10 +341,8 @@ pub async fn try_migrate_iam_config<S: StorageAPI>(store: Arc<S>) {
continue;
}
};
if let Err(e) = store
.put_object(RUSTFS_META_BUCKET, path, &mut PutObjReader::from_vec(data), &opts)
.await
{
let mut put_data = ChunkNativePutData::from_vec(data);
if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await {
warn!("write IAM config {path}: {e}");
} else {
info!("Migrated IAM config: {path}");
+9 -11
View File
@@ -16,7 +16,7 @@ use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, audit, notify, oidc, stor
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::global::is_first_cluster_node_local;
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use crate::store_api::{ChunkNativePutData, ObjectInfo, ObjectOptions, StorageAPI};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS};
@@ -128,10 +128,8 @@ pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()>
}
pub async fn save_config_with_opts<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
if let Err(err) = api
.put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data), opts)
.await
{
let mut put_data = ChunkNativePutData::from_vec(data);
if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
return Err(err);
}
@@ -1078,10 +1076,10 @@ mod tests {
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
use crate::set_disk::SetDisks;
use crate::store_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader,
HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations,
MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations,
ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
BucketInfo, BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, DeleteBucketOptions, DeletedObject,
GetObjectReader, HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info,
ListOperations, MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo,
ObjectOperations, ObjectOptions, ObjectToDelete, PartInfo, StorageAPI, WalkOptions,
};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
@@ -1304,7 +1302,7 @@ mod tests {
&self,
_bucket: &str,
_object: &str,
_data: &mut PutObjReader,
_data: &mut ChunkNativePutData,
_opts: &ObjectOptions,
) -> Result<ObjectInfo> {
panic!("unused in test")
@@ -1491,7 +1489,7 @@ mod tests {
_object: &str,
_upload_id: &str,
_part_id: usize,
_data: &mut PutObjReader,
_data: &mut ChunkNativePutData,
_opts: &ObjectOptions,
) -> Result<PartInfo> {
panic!("unused in test")
+39 -12
View File
@@ -150,18 +150,7 @@ impl Config {
return false;
}
let shard_size = shard_size as usize;
let mut inline_block = DEFAULT_INLINE_BLOCK;
if self.initialized {
inline_block = self.inline_block;
}
if versioned {
shard_size <= inline_block / 8
} else {
shard_size <= inline_block
}
shard_size as usize <= self.inline_shard_limit_bytes(versioned)
}
pub fn inline_block(&self) -> usize {
@@ -172,6 +161,15 @@ impl Config {
}
}
pub fn inline_shard_limit_bytes(&self, versioned: bool) -> usize {
let inline_block = self.inline_block();
if versioned { inline_block / 8 } else { inline_block }
}
pub fn inline_object_limit_bytes(&self, data_shards: usize, versioned: bool) -> usize {
self.inline_shard_limit_bytes(versioned).saturating_mul(data_shards.max(1))
}
pub fn capacity_optimized(&self) -> bool {
if !self.initialized {
false
@@ -336,3 +334,32 @@ pub fn validate_parity_inner(ss_parity: usize, rrs_parity: usize, set_drive_coun
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn inline_object_limit_matches_default_non_versioned_budget() {
let cfg = Config {
initialized: true,
inline_block: DEFAULT_INLINE_BLOCK,
..Default::default()
};
assert_eq!(cfg.inline_shard_limit_bytes(false), DEFAULT_INLINE_BLOCK);
assert_eq!(cfg.inline_object_limit_bytes(8, false), DEFAULT_INLINE_BLOCK * 8);
}
#[test]
fn inline_object_limit_scales_down_for_versioned_objects() {
let cfg = Config {
initialized: true,
inline_block: DEFAULT_INLINE_BLOCK,
..Default::default()
};
assert_eq!(cfg.inline_shard_limit_bytes(true), DEFAULT_INLINE_BLOCK / 8);
assert_eq!(cfg.inline_object_limit_bytes(8, true), DEFAULT_INLINE_BLOCK);
}
}
+15 -8
View File
@@ -14,9 +14,11 @@
use crate::error::{Error, Result};
use crate::store::ECStore;
use crate::store_api::{CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store_api::{
ChunkNativePutData, CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions,
};
use bytes::Bytes;
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use rustfs_rio::{BlockReadable, BoxReadBlockFuture, EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use std::io::Cursor;
use std::pin::Pin;
use std::sync::{
@@ -54,6 +56,11 @@ impl<R: AsyncRead + Unpin + Send + Sync> TryGetIndex for IndexedDataMovementRead
}
}
impl<R: AsyncRead + Unpin + Send + Sync> BlockReadable for IndexedDataMovementReader<R> {
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
Box::pin(rustfs_utils::read_full(self, buf))
}
}
pub fn decode_part_index(index: Option<&Bytes>) -> Option<Index> {
let bytes = index?;
let mut decoded = Index::new();
@@ -64,7 +71,7 @@ pub fn decode_part_index(index: Option<&Bytes>) -> Option<Index> {
}
}
pub fn put_obj_reader_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: Option<Index>) -> Result<PutObjReader> {
pub fn put_data_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: Option<Index>) -> Result<ChunkNativePutData> {
use sha2::{Digest, Sha256};
let sha256hex = if !chunk.is_empty() {
@@ -74,8 +81,8 @@ pub fn put_obj_reader_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, in
};
let reader = IndexedDataMovementReader::new(Cursor::new(chunk), index);
let hash_reader = HashReader::from_stream(reader, size, actual_size, None, sha256hex, false)?;
Ok(PutObjReader::new(hash_reader))
let hash_reader = HashReader::from_reader(reader, size, actual_size, None, sha256hex, false)?;
Ok(ChunkNativePutData::new(hash_reader))
}
pub fn new_multipart_abort_flag() -> Arc<AtomicBool> {
@@ -172,7 +179,7 @@ pub(crate) async fn migrate_object(
let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?;
let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size };
let index = decode_part_index(part.index.as_ref());
let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?;
let mut data = put_data_from_chunk(chunk, part_size, part_actual_size, index)?;
let pi = match store
.put_object_part(
@@ -254,8 +261,8 @@ pub(crate) async fn migrate_object(
.first()
.and_then(|part| decode_part_index(part.index.as_ref()));
let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index);
let hrd = HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?;
let mut data = PutObjReader::new(hrd);
let hrd = HashReader::from_reader(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?;
let mut data = ChunkNativePutData::new(hrd);
if let Err(err) = store
.put_object(
+9
View File
@@ -21,6 +21,7 @@ use crate::disk::{
use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_core::BoxChunkStream;
use std::{
path::PathBuf,
sync::{
@@ -738,6 +739,14 @@ impl DiskAPI for LocalDiskWrapper {
.await
}
async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<BoxChunkStream> {
self.track_disk_health(
|| async { self.disk.read_file_chunks(volume, path, offset, length).await },
get_max_timeout_duration(),
)
.await
}
async fn append_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileWriter> {
self.track_disk_health(|| async { self.disk.append_file(volume, path).await }, Duration::ZERO)
.await
+784 -74
View File
@@ -30,12 +30,19 @@ use crate::disk::{
};
use crate::erasure_coding::bitrot_verify;
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
use bytes::Bytes;
use bytes::{Bytes, BytesMut};
use futures_util::{StreamExt, stream};
use parking_lot::RwLock as ParkingLotRwLock;
use rustfs_config::{
DEFAULT_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES, DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES,
DEFAULT_OBJECT_ZERO_COPY_MODE, ENV_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES,
ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES, ENV_OBJECT_ZERO_COPY_MODE,
};
use rustfs_filemeta::{
Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
get_file_info, read_xl_meta_no_data,
};
use rustfs_io_core::{BoxChunkStream, BytesPool, IoChunk, MappedChunk, PooledChunk};
use rustfs_utils::HashAlgorithm;
use rustfs_utils::os::get_info;
use rustfs_utils::path::{
@@ -46,7 +53,7 @@ use std::collections::HashMap;
use std::collections::HashSet;
use std::fmt::Debug;
use std::io::SeekFrom;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use std::{
@@ -61,6 +68,11 @@ use tokio::time::interval;
use tracing::{debug, error, info, warn};
use uuid::Uuid;
#[cfg(test)]
use serial_test::serial;
#[cfg(test)]
use temp_env::with_var;
#[derive(Debug, Clone)]
pub struct FormatInfo {
pub id: Option<Uuid>,
@@ -97,6 +109,410 @@ pub struct LocalDisk {
exit_signal: Option<tokio::sync::broadcast::Sender<()>>,
}
const LOCAL_CHUNK_FAST_PATH_MIN_BYTES: usize = 64 * 1024;
const LOCAL_DISK_POOLED_SOURCE_FALLBACK: &str = "fallback";
const LOCAL_DISK_POOLED_SOURCE_COMPAT_COLLECT: &str = "compat_collect";
const LOCAL_DISK_POOLED_SOURCE_COMPAT_DIRECT: &str = "compat_direct";
const ACTIVE_MMAP_WINDOW_BUDGET_EXCEEDED_MESSAGE: &str = "active mmap window budget exceeded";
#[cfg(unix)]
const LOCAL_CHUNK_COMPAT_MAX_MAPPED_WINDOWS: usize = 1;
static LOCAL_CHUNK_FALLBACK_POOL: OnceLock<BytesPool> = OnceLock::new();
fn local_chunk_fallback_pool() -> &'static BytesPool {
LOCAL_CHUNK_FALLBACK_POOL.get_or_init(BytesPool::new_tiered)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LocalChunkZeroCopyMode {
Off,
Conservative,
Balanced,
Aggressive,
}
impl LocalChunkZeroCopyMode {
fn from_env() -> Self {
match rustfs_utils::get_env_str(ENV_OBJECT_ZERO_COPY_MODE, DEFAULT_OBJECT_ZERO_COPY_MODE)
.trim()
.to_ascii_lowercase()
.as_str()
{
"off" => Self::Off,
"conservative" => Self::Conservative,
"aggressive" => Self::Aggressive,
_ => Self::Balanced,
}
}
fn effective() -> Self {
if !rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE) {
return Self::Off;
}
Self::from_env()
}
const fn fast_path_min_bytes(self) -> usize {
match self {
Self::Aggressive => 1,
Self::Off | Self::Conservative | Self::Balanced => LOCAL_CHUNK_FAST_PATH_MIN_BYTES,
}
}
const fn allows_multi_window(self) -> bool {
matches!(self, Self::Balanced | Self::Aggressive)
}
const fn is_disabled(self) -> bool {
matches!(self, Self::Off)
}
}
#[cfg(unix)]
static ACTIVE_LOCAL_MMAP_BYTES: AtomicUsize = AtomicUsize::new(0);
#[cfg(unix)]
#[derive(Debug)]
struct ActiveMmapWindow {
mmap: memmap2::Mmap,
accounted_len: usize,
}
#[cfg(unix)]
impl AsRef<[u8]> for ActiveMmapWindow {
fn as_ref(&self) -> &[u8] {
&self.mmap[..]
}
}
#[cfg(unix)]
impl Drop for ActiveMmapWindow {
fn drop(&mut self) {
let remaining = ACTIVE_LOCAL_MMAP_BYTES
.fetch_sub(self.accounted_len, Ordering::AcqRel)
.saturating_sub(self.accounted_len);
rustfs_io_metrics::record_local_disk_active_mmap_bytes(remaining);
}
}
#[cfg(unix)]
#[allow(unsafe_code)]
fn mmap_page_size() -> usize {
static PAGE_SIZE: OnceLock<usize> = OnceLock::new();
*PAGE_SIZE.get_or_init(|| {
let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
if page_size <= 0 { 4096 } else { page_size as usize }
})
}
#[cfg(unix)]
fn configured_local_chunk_window_bytes() -> usize {
let page_size = mmap_page_size();
rustfs_utils::get_env_usize(ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES, DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES)
.max(page_size)
.div_ceil(page_size)
* page_size
}
#[cfg(unix)]
fn configured_local_chunk_max_active_mmap_bytes() -> usize {
rustfs_utils::get_env_usize(ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES, DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES)
.max(configured_local_chunk_window_bytes())
}
#[cfg(unix)]
fn should_prefer_pooled_zero_copy_compat(mode: LocalChunkZeroCopyMode, length: usize, window_bytes: usize) -> bool {
if mode.is_disabled() || !mode.allows_multi_window() || length < mode.fast_path_min_bytes() || window_bytes == 0 {
return false;
}
length.div_ceil(window_bytes) > LOCAL_CHUNK_COMPAT_MAX_MAPPED_WINDOWS
}
fn fallback_reason_for_local_mmap_error(err: &DiskError) -> rustfs_io_metrics::FallbackReason {
match err {
DiskError::Io(io_error) if io_error.to_string().contains(ACTIVE_MMAP_WINDOW_BUDGET_EXCEEDED_MESSAGE) => {
rustfs_io_metrics::FallbackReason::WindowLimitExceeded
}
_ => rustfs_io_metrics::FallbackReason::MmapUnavailable,
}
}
#[cfg(unix)]
fn try_reserve_active_mmap_bytes(accounted_len: usize, max_active_bytes: usize) -> bool {
loop {
let current = ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire);
let Some(next) = current.checked_add(accounted_len) else {
return false;
};
if next > max_active_bytes {
return false;
}
if ACTIVE_LOCAL_MMAP_BYTES
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
rustfs_io_metrics::record_local_disk_active_mmap_bytes(next);
return true;
}
}
}
#[cfg(unix)]
#[allow(unsafe_code)]
fn map_file_region_bytes(file_path: &Path, offset: usize, length: usize, max_active_bytes: usize) -> Result<Bytes> {
use memmap2::MmapOptions;
let aligned_offset = offset / mmap_page_size() * mmap_page_size();
let logical_offset = offset - aligned_offset;
let map_length = logical_offset.checked_add(length).ok_or(DiskError::FileCorrupt)?;
let visible_end = logical_offset.checked_add(length).ok_or(DiskError::FileCorrupt)?;
if !try_reserve_active_mmap_bytes(map_length, max_active_bytes) {
return Err(DiskError::other(ACTIVE_MMAP_WINDOW_BUDGET_EXCEEDED_MESSAGE));
}
let file = std::fs::File::open(file_path).map_err(DiskError::from)?;
let mmap_result =
unsafe { MmapOptions::new().offset(aligned_offset as u64).len(map_length).map(&file) }.map_err(DiskError::other);
let mmap = match mmap_result {
Ok(mmap) => mmap,
Err(err) => {
let remaining = ACTIVE_LOCAL_MMAP_BYTES
.fetch_sub(map_length, Ordering::AcqRel)
.saturating_sub(map_length);
rustfs_io_metrics::record_local_disk_active_mmap_bytes(remaining);
return Err(err);
}
};
let bytes = Bytes::from_owner(ActiveMmapWindow {
mmap,
accounted_len: map_length,
});
Ok(bytes.slice(logical_offset..visible_end))
}
#[cfg(unix)]
#[allow(unsafe_code)]
fn map_file_region_chunk(file_path: &Path, offset: usize, length: usize, max_active_bytes: usize) -> Result<MappedChunk> {
let bytes = map_file_region_bytes(file_path, offset, length, max_active_bytes)?;
MappedChunk::new(bytes, 0, length).map_err(DiskError::other)
}
#[cfg(unix)]
#[derive(Debug)]
struct LocalMappedChunkStreamState {
file_path: PathBuf,
next_offset: usize,
remaining: usize,
window_bytes: usize,
}
async fn read_file_pooled_chunk_from_path(file_path: PathBuf, offset: usize, length: usize) -> std::io::Result<IoChunk> {
read_file_pooled_chunk_from_path_with_source(file_path, offset, length, LOCAL_DISK_POOLED_SOURCE_FALLBACK).await
}
async fn read_file_pooled_chunk_from_path_with_source(
file_path: PathBuf,
offset: usize,
length: usize,
metric_source: &'static str,
) -> std::io::Result<IoChunk> {
let mut file = File::open(file_path).await?;
if offset > 0 {
file.seek(SeekFrom::Start(offset as u64)).await?;
}
let mut buffer = local_chunk_fallback_pool().acquire_buffer(length).await;
buffer.resize(length, 0);
file.read_exact(&mut buffer[..length]).await?;
rustfs_io_metrics::record_local_disk_pooled_chunk(metric_source, length);
Ok(IoChunk::Pooled(PooledChunk::new(buffer, length).map_err(std::io::Error::other)?))
}
async fn prepare_read_file_request(disk: &LocalDisk, volume: &str, path: &str) -> Result<(PathBuf, PathBuf, Metadata)> {
let volume_dir = disk.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
access(&volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
let file_path = disk.get_object_path(volume, path)?;
check_path_length(file_path.to_string_lossy().as_ref())?;
let file_path_clone = file_path.clone();
let meta = tokio::task::spawn_blocking(move || std::fs::metadata(&file_path_clone).map_err(DiskError::from))
.await
.map_err(DiskError::from)??;
Ok((volume_dir, file_path, meta))
}
fn validate_read_file_bounds(meta: &Metadata, offset: usize, length: usize) -> Result<()> {
let end_offset = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?;
if meta.len() < end_offset as u64 {
error!(
"read_file: file size is less than offset + length {} + {} = {}",
offset,
length,
meta.len()
);
return Err(DiskError::FileCorrupt);
}
Ok(())
}
#[cfg(unix)]
fn build_lazy_mapped_chunk_stream(
file_path: PathBuf,
offset: usize,
length: usize,
window_bytes: usize,
max_active_bytes: usize,
) -> BoxChunkStream {
let state = LocalMappedChunkStreamState {
file_path,
next_offset: offset,
remaining: length,
window_bytes,
};
Box::pin(stream::unfold(Some(state), move |state| async move {
let mut state = match state {
Some(state) => state,
None => return None,
};
if state.remaining == 0 {
return None;
}
let visible_len = state.remaining.min(state.window_bytes);
let window_offset = state.next_offset;
let file_path = state.file_path.clone();
let mmap_result =
tokio::task::spawn_blocking(move || map_file_region_chunk(&file_path, window_offset, visible_len, max_active_bytes))
.await;
match mmap_result {
Ok(Ok(chunk)) => {
state.next_offset += visible_len;
state.remaining -= visible_len;
let next_state = if state.remaining == 0 { None } else { Some(state) };
Some((Ok(IoChunk::Mapped(chunk)), next_state))
}
Ok(Err(err)) => {
rustfs_io_metrics::record_io_fallback(
rustfs_io_metrics::IoStage::LocalDiskChunk,
fallback_reason_for_local_mmap_error(&err),
);
debug!(
error = %err,
offset = window_offset,
len = visible_len,
"local disk lazy mmap window failed, falling back to buffered remainder"
);
let fallback =
read_file_pooled_chunk_from_path(state.file_path.clone(), state.next_offset, state.remaining).await;
Some((fallback, None))
}
Err(err) => {
rustfs_io_metrics::record_io_fallback(
rustfs_io_metrics::IoStage::LocalDiskChunk,
rustfs_io_metrics::FallbackReason::MmapUnavailable,
);
debug!(
error = %err,
offset = window_offset,
len = visible_len,
"local disk lazy mmap task failed, falling back to buffered remainder"
);
let fallback =
read_file_pooled_chunk_from_path(state.file_path.clone(), state.next_offset, state.remaining).await;
Some((fallback, None))
}
}
}))
}
async fn read_file_pooled_chunk_fallback(
disk: &LocalDisk,
volume_dir: &Path,
file_path: PathBuf,
offset: usize,
length: usize,
) -> Result<IoChunk> {
read_file_pooled_chunk_fallback_with_source(disk, volume_dir, file_path, offset, length, LOCAL_DISK_POOLED_SOURCE_FALLBACK)
.await
}
async fn read_file_pooled_chunk_fallback_with_source(
disk: &LocalDisk,
volume_dir: &Path,
file_path: PathBuf,
offset: usize,
length: usize,
metric_source: &'static str,
) -> Result<IoChunk> {
let mut f = disk.open_file(file_path, O_RDONLY, volume_dir).await?;
if offset > 0 {
f.seek(SeekFrom::Start(offset as u64)).await?;
}
let mut buffer = local_chunk_fallback_pool().acquire_buffer(length).await;
buffer.resize(length, 0);
f.read_exact(&mut buffer[..length]).await?;
rustfs_io_metrics::record_local_disk_pooled_chunk(metric_source, length);
Ok(IoChunk::Pooled(PooledChunk::new(buffer, length).map_err(DiskError::other)?))
}
async fn collect_chunk_stream_bytes(mut stream: BoxChunkStream, expected_len: usize) -> Result<Bytes> {
let Some(first) = stream.next().await else {
return Ok(Bytes::new());
};
let first = first.map_err(DiskError::from)?;
let first_len = first.len();
if matches!(first, IoChunk::Pooled(_)) {
rustfs_io_metrics::record_local_disk_pooled_chunk(LOCAL_DISK_POOLED_SOURCE_COMPAT_COLLECT, first_len);
}
let first_bytes = first.as_bytes();
let Some(second) = stream.next().await else {
return Ok(first_bytes);
};
let second = second.map_err(DiskError::from)?;
let second_len = second.len();
if matches!(second, IoChunk::Pooled(_)) {
rustfs_io_metrics::record_local_disk_pooled_chunk(LOCAL_DISK_POOLED_SOURCE_COMPAT_COLLECT, second_len);
}
let mut chunk_count = 2usize;
let mut total_bytes = first_len + second_len;
let mut buffer = BytesMut::with_capacity(expected_len);
buffer.extend_from_slice(first_bytes.as_ref());
buffer.extend_from_slice(second.as_bytes().as_ref());
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(DiskError::from)?;
let chunk_len = chunk.len();
if matches!(chunk, IoChunk::Pooled(_)) {
rustfs_io_metrics::record_local_disk_pooled_chunk(LOCAL_DISK_POOLED_SOURCE_COMPAT_COLLECT, chunk_len);
}
chunk_count += 1;
total_bytes += chunk_len;
buffer.extend_from_slice(chunk.as_bytes().as_ref());
}
rustfs_io_metrics::record_local_disk_compat_collect(chunk_count, total_bytes);
Ok(buffer.freeze())
}
impl Drop for LocalDisk {
fn drop(&mut self) {
if let Some(exit_signal) = self.exit_signal.take() {
@@ -1835,87 +2251,96 @@ impl DiskAPI for LocalDisk {
use std::time::Instant;
let start = Instant::now();
let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
access(&volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
let file_path = self.get_object_path(volume, path)?;
check_path_length(file_path.to_string_lossy().as_ref())?;
// Verify file exists and get metadata
let file_path_clone = file_path.clone();
let meta = tokio::task::spawn_blocking(move || std::fs::metadata(&file_path_clone).map_err(DiskError::from))
.await
.map_err(DiskError::from)??;
let end_offset = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?;
if meta.len() < end_offset as u64 {
error!(
"read_file_zero_copy: file size is less than offset + length {} + {} = {}",
offset,
length,
meta.len()
);
return Err(DiskError::FileCorrupt);
}
// Unix: use mmap to read the data (copies into Bytes for safe ownership)
// Non-Unix: fall back to efficient read
#[cfg(unix)]
{
use memmap2::MmapOptions;
let file_path_clone = file_path.clone();
let offset_u64 = offset as u64;
let bytes = tokio::task::spawn_blocking(move || {
let file = std::fs::File::open(&file_path_clone).map_err(DiskError::from)?;
// Create memory map for the specified region
// SAFETY: The file is opened as read-only, and we're mapping a region
// that we've already verified exists and is within file bounds.
let mmap = unsafe { MmapOptions::new().offset(offset_u64).len(length).map(&file) }.map_err(DiskError::other)?;
// Copy the mapped region into a Bytes buffer. This avoids undefined
// behavior from treating OS-managed mmap memory as allocator-managed
// Vec storage, at the cost of an extra copy.
Ok::<Bytes, DiskError>(Bytes::copy_from_slice(&mmap))
})
.await
.map_err(DiskError::from)??;
// Log successful mmap read metrics
let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
// Record mmap read metrics
rustfs_io_metrics::record_zero_copy_read(length, duration_ms);
debug!(size = length, duration_ms = duration_ms, "mmap_read_success");
return Ok(bytes);
let zero_copy_mode = LocalChunkZeroCopyMode::effective();
let window_bytes = configured_local_chunk_window_bytes();
if should_prefer_pooled_zero_copy_compat(zero_copy_mode, length, window_bytes) {
let (volume_dir, file_path, meta) = prepare_read_file_request(self, volume, path).await?;
validate_read_file_bounds(&meta, offset, length)?;
let chunk = read_file_pooled_chunk_fallback_with_source(
self,
&volume_dir,
file_path,
offset,
length,
LOCAL_DISK_POOLED_SOURCE_COMPAT_DIRECT,
)
.await?;
let bytes = collect_chunk_stream_bytes(Box::pin(stream::iter(vec![Ok(chunk)])), length).await?;
debug!(
size = bytes.len(),
duration_ms = start.elapsed().as_secs_f64() * 1000.0,
"chunk_compat_read_pooled_success"
);
return Ok(bytes);
}
}
// Non-Unix fallback: efficient read into Bytes
#[cfg(not(unix))]
let bytes = collect_chunk_stream_bytes(self.read_file_chunks(volume, path, offset, length).await?, length).await?;
debug!(
size = bytes.len(),
duration_ms = start.elapsed().as_secs_f64() * 1000.0,
"chunk_compat_read_success"
);
Ok(bytes)
}
#[allow(unsafe_code)]
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<BoxChunkStream> {
let (volume_dir, file_path, meta) = prepare_read_file_request(self, volume, path).await?;
validate_read_file_bounds(&meta, offset, length)?;
let zero_copy_mode = LocalChunkZeroCopyMode::effective();
if zero_copy_mode.is_disabled() {
rustfs_io_metrics::record_io_fallback(
rustfs_io_metrics::IoStage::LocalDiskChunk,
rustfs_io_metrics::FallbackReason::MmapDisabled,
);
let chunk = read_file_pooled_chunk_fallback(self, &volume_dir, file_path, offset, length).await?;
return Ok(Box::pin(stream::iter(vec![Ok(chunk)])));
}
if length < zero_copy_mode.fast_path_min_bytes() {
rustfs_io_metrics::record_io_fallback(
rustfs_io_metrics::IoStage::LocalDiskChunk,
rustfs_io_metrics::FallbackReason::SmallObject,
);
let chunk = read_file_pooled_chunk_fallback(self, &volume_dir, file_path, offset, length).await?;
return Ok(Box::pin(stream::iter(vec![Ok(chunk)])));
}
#[cfg(unix)]
{
// Record zero-copy fallback
rustfs_io_metrics::record_zero_copy_fallback("non_unix_platform");
let window_bytes = configured_local_chunk_window_bytes();
debug!(reason = "non_unix_platform", "zero_copy_fallback");
let mut f = self.open_file(file_path, O_RDONLY, volume_dir).await?;
if offset > 0 {
f.seek(SeekFrom::Start(offset as u64)).await?;
if !zero_copy_mode.allows_multi_window() && length > window_bytes {
rustfs_io_metrics::record_io_fallback(
rustfs_io_metrics::IoStage::LocalDiskChunk,
rustfs_io_metrics::FallbackReason::WindowLimitExceeded,
);
let chunk = read_file_pooled_chunk_fallback(self, &volume_dir, file_path, offset, length).await?;
return Ok(Box::pin(stream::iter(vec![Ok(chunk)])));
}
let mut buffer = Vec::with_capacity(length);
buffer.resize(length, 0);
f.read_exact(&mut buffer).await?;
return Ok(build_lazy_mapped_chunk_stream(
file_path,
offset,
length,
window_bytes,
configured_local_chunk_max_active_mmap_bytes(),
));
}
Ok(Bytes::from(buffer))
#[cfg(not(unix))]
{
rustfs_io_metrics::record_io_fallback(
rustfs_io_metrics::IoStage::LocalDiskChunk,
rustfs_io_metrics::FallbackReason::MmapUnavailable,
);
let chunk = read_file_pooled_chunk_fallback(self, &volume_dir, file_path, offset, length).await?;
Ok(Box::pin(stream::iter(vec![Ok(chunk)])))
}
}
@@ -2674,6 +3099,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
#[cfg(test)]
mod test {
use super::*;
use futures_util::StreamExt;
#[tokio::test]
async fn test_skip_access_checks() {
@@ -2960,6 +3386,22 @@ mod test {
assert!(matches!(result, Err(DiskError::FileCorrupt)));
}
#[tokio::test]
async fn test_read_file_zero_copy_supports_non_zero_offset() {
use tempfile::tempdir;
let dir = tempdir().unwrap();
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
disk.make_volume("test-volume").await.unwrap();
let content = Bytes::from_static(b"0123456789abcdef");
disk.write_all("test-volume", "test-file.txt", content.clone()).await.unwrap();
let result = disk.read_file_zero_copy("test-volume", "test-file.txt", 3, 7).await.unwrap();
assert_eq!(result, Bytes::from_static(b"3456789"));
}
#[test]
fn test_is_valid_volname() {
// Valid volume names (length >= 3)
@@ -3057,6 +3499,274 @@ mod test {
let _ = fs::remove_file(test_file).await;
}
#[tokio::test]
#[serial]
async fn test_read_file_chunks_returns_pooled_chunk_for_local_fallback() {
let dir = tempfile::tempdir().unwrap();
let bucket = "chunk-bucket";
let object = "obj.txt";
let content = b"chunk-data";
fs::create_dir_all(dir.path().join(bucket)).await.unwrap();
fs::write(dir.path().join(bucket).join(object), content).await.unwrap();
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
let mut stream = disk.read_file_chunks(bucket, object, 0, content.len()).await.unwrap();
let first = stream.next().await.unwrap().unwrap();
assert!(matches!(first, IoChunk::Pooled(_)));
assert_eq!(first.as_bytes(), Bytes::from_static(content));
assert!(stream.next().await.is_none());
}
#[tokio::test]
#[serial]
async fn test_read_file_chunks_prefers_mapped_chunk_when_eligible() {
let dir = tempfile::tempdir().unwrap();
let bucket = "chunk-bucket";
let object = "obj-large.txt";
let content = vec![7u8; LOCAL_CHUNK_FAST_PATH_MIN_BYTES];
fs::create_dir_all(dir.path().join(bucket)).await.unwrap();
fs::write(dir.path().join(bucket).join(object), &content).await.unwrap();
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
let mut stream = disk.read_file_chunks(bucket, object, 0, content.len()).await.unwrap();
let first = stream.next().await.unwrap().unwrap();
#[cfg(unix)]
assert!(matches!(first, IoChunk::Mapped(_)));
#[cfg(not(unix))]
assert!(matches!(first, IoChunk::Pooled(_)));
assert_eq!(first.as_bytes(), Bytes::from(content));
assert!(stream.next().await.is_none());
}
#[tokio::test]
#[serial]
async fn test_read_file_chunks_falls_back_to_pooled_for_small_object() {
let dir = tempfile::tempdir().unwrap();
let bucket = "chunk-bucket";
let object = "obj-small.txt";
let content = b"small-object";
fs::create_dir_all(dir.path().join(bucket)).await.unwrap();
fs::write(dir.path().join(bucket).join(object), content).await.unwrap();
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
let mut stream = disk.read_file_chunks(bucket, object, 0, content.len()).await.unwrap();
let first = stream.next().await.unwrap().unwrap();
assert!(matches!(first, IoChunk::Pooled(_)));
assert_eq!(first.as_bytes(), Bytes::from_static(content));
assert!(stream.next().await.is_none());
}
#[tokio::test]
#[serial]
async fn test_read_file_chunks_supports_non_zero_offset() {
let dir = tempfile::tempdir().unwrap();
let bucket = "chunk-bucket";
let object = "obj-offset.txt";
let content = vec![3u8; LOCAL_CHUNK_FAST_PATH_MIN_BYTES + 16];
fs::create_dir_all(dir.path().join(bucket)).await.unwrap();
fs::write(dir.path().join(bucket).join(object), &content).await.unwrap();
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
let mut stream = disk
.read_file_chunks(bucket, object, 1, LOCAL_CHUNK_FAST_PATH_MIN_BYTES)
.await
.unwrap();
let first = stream.next().await.unwrap().unwrap();
#[cfg(unix)]
assert!(matches!(first, IoChunk::Mapped(_)));
#[cfg(not(unix))]
assert!(matches!(first, IoChunk::Pooled(_)));
assert_eq!(first.as_bytes(), Bytes::copy_from_slice(&content[1..1 + LOCAL_CHUNK_FAST_PATH_MIN_BYTES]));
assert!(stream.next().await.is_none());
}
#[cfg(unix)]
#[tokio::test]
#[serial]
async fn test_read_file_chunks_splits_large_reads_into_multiple_windows() {
let dir = tempfile::tempdir().unwrap();
let bucket = "chunk-bucket";
let object = "obj-windowed.txt";
let content = vec![5u8; DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES + 32];
fs::create_dir_all(dir.path().join(bucket)).await.unwrap();
fs::write(dir.path().join(bucket).join(object), &content).await.unwrap();
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
let mut stream = disk.read_file_chunks(bucket, object, 0, content.len()).await.unwrap();
let first = stream.next().await.unwrap().unwrap();
let second = stream.next().await.unwrap().unwrap();
assert!(matches!(first, IoChunk::Mapped(_)));
assert!(matches!(second, IoChunk::Mapped(_)));
assert_eq!(first.len(), DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES);
assert_eq!(second.len(), 32);
assert_eq!(first.as_bytes(), Bytes::from(vec![5u8; DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES]));
assert_eq!(second.as_bytes(), Bytes::from(vec![5u8; 32]));
assert!(stream.next().await.is_none());
}
#[cfg(unix)]
#[tokio::test]
#[serial]
async fn test_read_file_zero_copy_collects_multi_window_chunk_stream() {
let dir = tempfile::tempdir().unwrap();
let bucket = "chunk-bucket";
let object = "obj-zero-copy-compat.txt";
let content = vec![6u8; DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES + 48];
fs::create_dir_all(dir.path().join(bucket)).await.unwrap();
fs::write(dir.path().join(bucket).join(object), &content).await.unwrap();
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
let bytes = disk.read_file_zero_copy(bucket, object, 0, content.len()).await.unwrap();
assert_eq!(bytes, Bytes::from(content));
}
#[cfg(unix)]
#[test]
#[serial]
fn test_read_file_chunks_lazy_windows_reuse_single_window_budget() {
let page_size = mmap_page_size();
let window_bytes = page_size.to_string();
let max_active_bytes = page_size.to_string();
with_var(ENV_OBJECT_ZERO_COPY_ENABLE, Some("true"), || {
with_var(ENV_OBJECT_ZERO_COPY_MODE, Some("aggressive"), || {
with_var(ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES, Some(window_bytes.clone()), || {
with_var(ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES, Some(max_active_bytes.clone()), || {
ACTIVE_LOCAL_MMAP_BYTES.store(0, Ordering::Release);
let runtime = tokio::runtime::Runtime::new().unwrap();
runtime.block_on(async {
let dir = tempfile::tempdir().unwrap();
let bucket = "chunk-bucket";
let object = "obj-budgeted.txt";
let content = vec![9u8; page_size * 2];
fs::create_dir_all(dir.path().join(bucket)).await.unwrap();
fs::write(dir.path().join(bucket).join(object), &content).await.unwrap();
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
let mut stream = disk.read_file_chunks(bucket, object, 0, content.len()).await.unwrap();
let first = stream.next().await.unwrap().unwrap();
assert!(matches!(first, IoChunk::Mapped(_)));
assert_eq!(first.len(), page_size);
assert_eq!(ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire), page_size);
drop(first);
assert_eq!(ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire), 0);
let second = stream.next().await.unwrap().unwrap();
assert!(matches!(second, IoChunk::Mapped(_)));
assert_eq!(second.len(), page_size);
assert_eq!(ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire), page_size);
drop(second);
assert_eq!(ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire), 0);
assert!(stream.next().await.is_none());
});
assert_eq!(ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire), 0);
});
});
});
});
}
#[test]
#[serial]
fn test_local_chunk_zero_copy_mode_respects_enable_and_mode_env() {
with_var(ENV_OBJECT_ZERO_COPY_ENABLE, Some("false"), || {
assert_eq!(LocalChunkZeroCopyMode::effective(), LocalChunkZeroCopyMode::Off);
});
with_var(ENV_OBJECT_ZERO_COPY_ENABLE, Some("true"), || {
with_var(ENV_OBJECT_ZERO_COPY_MODE, Some("aggressive"), || {
assert_eq!(LocalChunkZeroCopyMode::effective(), LocalChunkZeroCopyMode::Aggressive);
});
});
}
#[cfg(unix)]
#[test]
#[serial]
fn test_should_prefer_pooled_zero_copy_compat_for_multi_window_requests() {
let window_bytes = 1024;
let balanced_multi_window_len = LOCAL_CHUNK_FAST_PATH_MIN_BYTES.max(window_bytes * 2);
assert!(!should_prefer_pooled_zero_copy_compat(
LocalChunkZeroCopyMode::Off,
window_bytes * 2,
window_bytes
));
assert!(!should_prefer_pooled_zero_copy_compat(
LocalChunkZeroCopyMode::Conservative,
window_bytes * 2,
window_bytes
));
assert!(!should_prefer_pooled_zero_copy_compat(
LocalChunkZeroCopyMode::Balanced,
window_bytes,
window_bytes
));
assert!(should_prefer_pooled_zero_copy_compat(
LocalChunkZeroCopyMode::Balanced,
balanced_multi_window_len,
window_bytes
));
assert!(should_prefer_pooled_zero_copy_compat(
LocalChunkZeroCopyMode::Aggressive,
window_bytes * 2,
window_bytes
));
}
#[test]
fn test_fallback_reason_for_local_mmap_error_distinguishes_budget_limit() {
let budget_err = DiskError::other(ACTIVE_MMAP_WINDOW_BUDGET_EXCEEDED_MESSAGE);
let generic_err = DiskError::other("mmap failed");
assert_eq!(
fallback_reason_for_local_mmap_error(&budget_err),
rustfs_io_metrics::FallbackReason::WindowLimitExceeded
);
assert_eq!(
fallback_reason_for_local_mmap_error(&generic_err),
rustfs_io_metrics::FallbackReason::MmapUnavailable
);
}
#[cfg(unix)]
#[test]
#[serial]
fn test_configured_local_chunk_window_bytes_aligns_to_page_size() {
with_var(ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES, Some("12345"), || {
let page_size = mmap_page_size();
let window_bytes = configured_local_chunk_window_bytes();
assert!(window_bytes >= 12345);
assert_eq!(window_bytes % page_size, 0);
});
}
#[test]
fn test_is_root_path() {
// Unix root path
+12
View File
@@ -41,6 +41,7 @@ use error::DiskError;
use error::{Error, Result};
use local::LocalDisk;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_core::BoxChunkStream;
use rustfs_madmin::info_commands::DiskMetrics;
use serde::{Deserialize, Serialize};
use std::{fmt::Debug, path::PathBuf, sync::Arc};
@@ -295,6 +296,14 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(skip(self))]
async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<BoxChunkStream> {
match self {
Disk::Local(local_disk) => local_disk.read_file_chunks(volume, path, offset, length).await,
Disk::Remote(remote_disk) => remote_disk.read_file_chunks(volume, path, offset, length).await,
}
}
#[tracing::instrument(skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
match self {
@@ -505,6 +514,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
/// On other platforms, falls back to efficient read operations.
async fn read_file_zero_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Bytes>;
/// Chunk-based file read compatibility layer for the zero-copy data plane.
async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<BoxChunkStream>;
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter>;
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result<FileWriter>;
// ReadFileStream
@@ -172,6 +172,52 @@ where
}
}
impl BitrotWriter<CustomWriter> {
fn write_inline_sync(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if buf.is_empty() {
return Ok(0);
}
if self.finished {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "bitrot writer already finished"));
}
if buf.len() > self.shard_size {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("data size {} exceeds shard size {}", buf.len(), self.shard_size),
));
}
if buf.len() < self.shard_size {
self.finished = true;
}
match &mut self.inner {
CustomWriter::InlineBuffer(data) => {
if self.hash_algo.size() > 0 {
let hash = self.hash_algo.hash_encode(buf);
if hash.as_ref().is_empty() {
error!("bitrot writer write hash error: hash is empty");
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "hash is empty"));
}
data.extend_from_slice(hash.as_ref());
}
data.extend_from_slice(buf);
Ok(buf.len())
}
CustomWriter::Other(_) => Err(std::io::Error::other("inline sync write requires inline buffer writer")),
}
}
fn shutdown_inline_sync(&mut self) -> std::io::Result<()> {
match self.inner {
CustomWriter::InlineBuffer(_) => Ok(()),
CustomWriter::Other(_) => Err(std::io::Error::other("inline sync shutdown requires inline buffer writer")),
}
}
}
async fn write_all_vectored<W>(writer: &mut W, hash: &[u8], data: &[u8]) -> std::io::Result<()>
where
W: AsyncWrite + Unpin,
@@ -280,6 +326,10 @@ impl CustomWriter {
Self::Other(_) => None,
}
}
pub fn is_inline_buffer(&self) -> bool {
matches!(self, Self::InlineBuffer(_))
}
}
impl AsyncWrite for CustomWriter {
@@ -397,6 +447,24 @@ impl BitrotWriterWrapper {
self.bitrot_writer.shutdown().await
}
pub fn is_inline_buffer(&self) -> bool {
matches!(self.writer_type, WriterType::InlineBuffer)
}
pub fn write_inline_sync(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if !self.is_inline_buffer() {
return Err(std::io::Error::other("inline sync write requires inline buffer writer"));
}
self.bitrot_writer.write_inline_sync(buf)
}
pub fn shutdown_inline_sync(&mut self) -> std::io::Result<()> {
if !self.is_inline_buffer() {
return Err(std::io::Error::other("inline sync shutdown requires inline buffer writer"));
}
self.bitrot_writer.shutdown_inline_sync()
}
/// Extract the inline buffer data, consuming the wrapper
pub fn into_inline_data(self) -> Option<Vec<u8>> {
match self.writer_type {
+242 -3
View File
@@ -17,6 +17,7 @@ use crate::disk::error_reduce::reduce_errs;
use crate::erasure_coding::{BitrotReader, Erasure};
use futures::stream::{FuturesUnordered, StreamExt};
use pin_project_lite::pin_project;
use rustfs_io_core::{IoChunk, PooledChunk};
use std::io;
use std::io::ErrorKind;
use tokio::io::AsyncRead;
@@ -155,6 +156,68 @@ fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
size
}
fn block_window(
offset: usize,
length: usize,
block_size: usize,
block_index: usize,
start_block: usize,
end_block: usize,
) -> (usize, usize) {
let end_remainder = offset.saturating_add(length) % block_size;
if start_block == end_block {
(offset % block_size, length)
} else if block_index == start_block {
(offset % block_size, block_size - (offset % block_size))
} else if block_index == end_block {
(0, if end_remainder == 0 { block_size } else { end_remainder })
} else {
(0, block_size)
}
}
fn take_data_blocks_as_chunks(
shards: &mut [Option<Vec<u8>>],
data_blocks: usize,
mut offset: usize,
length: usize,
) -> io::Result<Vec<IoChunk>> {
if get_data_block_len(shards, data_blocks) < length {
error!("take_data_blocks_as_chunks get_data_block_len < length");
return Err(io::Error::new(ErrorKind::UnexpectedEof, "Not enough data blocks to write"));
}
let mut chunks = Vec::new();
let mut remaining = length;
for block_op in shards.iter_mut().take(data_blocks) {
let Some(block) = block_op.take() else {
error!("take_data_blocks_as_chunks block_op.is_none()");
return Err(io::Error::new(ErrorKind::UnexpectedEof, "Missing data block"));
};
if offset >= block.len() {
offset -= block.len();
continue;
}
let start = offset;
offset = 0;
let take = (block.len() - start).min(remaining);
let chunk = if start == 0 && take == block.len() {
IoChunk::Pooled(PooledChunk::from_vec(block))
} else {
IoChunk::Pooled(PooledChunk::from_vec(block).slice(start, take)?)
};
chunks.push(chunk);
remaining -= take;
if remaining == 0 {
break;
}
}
Ok(chunks)
}
/// Write data blocks from encoded blocks to target, supporting offset and length
async fn write_data_blocks<W>(
writer: &mut W,
@@ -213,6 +276,134 @@ where
Ok(total_written)
}
pub(crate) struct ErasureChunkDecoder<R> {
erasure: Erasure,
reader: ParallelReader<R>,
offset: usize,
length: usize,
start_block: usize,
end_block: usize,
current_block: usize,
written: usize,
healable_error: Option<Error>,
finished: bool,
}
impl<R> ErasureChunkDecoder<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
pub(crate) fn new(
erasure: Erasure,
readers: Vec<Option<BitrotReader<R>>>,
offset: usize,
length: usize,
total_length: usize,
) -> io::Result<Self> {
if readers.len() != erasure.data_shards + erasure.parity_shards {
return Err(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers"));
}
let end_offset = offset
.checked_add(length)
.ok_or_else(|| io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"))?;
if end_offset > total_length {
return Err(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"));
}
let start_block = offset / erasure.block_size;
let end_block = if length == 0 {
start_block
} else {
end_offset.saturating_sub(1) / erasure.block_size
};
let reader = ParallelReader::new(readers, erasure.clone(), offset, total_length);
Ok(Self {
erasure,
reader,
offset,
length,
start_block,
end_block,
current_block: start_block,
written: 0,
healable_error: None,
finished: length == 0,
})
}
pub(crate) async fn next_chunks(&mut self) -> io::Result<Option<Vec<IoChunk>>> {
if self.finished {
return Ok(None);
}
if self.current_block > self.end_block {
self.finished = true;
return Ok(None);
}
let block_index = self.current_block;
self.current_block += 1;
let (block_offset, block_length) = block_window(
self.offset,
self.length,
self.erasure.block_size,
block_index,
self.start_block,
self.end_block,
);
if block_length == 0 {
self.finished = true;
return Ok(None);
}
let (mut shards, errs) = self.reader.read().await;
if self.healable_error.is_none()
&& let (_, Some(err)) = reduce_errs(&errs, &[])
&& (err == Error::FileNotFound || err == Error::FileCorrupt)
{
self.healable_error = Some(err);
}
if !self.reader.can_decode(&shards) {
self.finished = true;
error!("reconstructed chunk decoder can_decode errs: {:?}", &errs);
return Err(Error::ErasureReadQuorum.into());
}
if let Err(err) = self.erasure.decode_data(&mut shards) {
self.finished = true;
error!("reconstructed chunk decoder decode_data err: {:?}", err);
return Err(err);
}
let chunks = take_data_blocks_as_chunks(&mut shards, self.erasure.data_shards, block_offset, block_length)?;
self.written += chunks.iter().map(IoChunk::len).sum::<usize>();
Ok(Some(chunks))
}
pub(crate) fn written(&self) -> usize {
self.written
}
pub(crate) fn finish_error(&self) -> Option<io::Error> {
if self.written < self.length {
Some(Error::LessData.into())
} else {
None
}
}
pub(crate) fn take_healable_error(&mut self) -> Option<Error> {
self.healable_error.take()
}
}
pub(crate) type ReconstructedChunkDecoder<R> = ErasureChunkDecoder<R>;
impl Erasure {
pub async fn decode<W, R>(
&self,
@@ -230,7 +421,10 @@ impl Erasure {
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")));
}
if offset + length > total_length {
let Some(end_offset) = offset.checked_add(length) else {
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
};
if end_offset > total_length {
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
}
@@ -245,7 +439,7 @@ impl Erasure {
let mut reader = ParallelReader::new(readers, self.clone(), offset, total_length);
let start = offset / self.block_size;
let end = (offset + length) / self.block_size;
let end = end_offset.saturating_sub(1) / self.block_size;
for i in start..=end {
let (block_offset, block_length) = if start == end {
@@ -253,7 +447,8 @@ impl Erasure {
} else if i == start {
(offset % self.block_size, self.block_size - (offset % self.block_size))
} else if i == end {
(0, (offset + length) % self.block_size)
let end_remainder = end_offset % self.block_size;
(0, if end_remainder == 0 { self.block_size } else { end_remainder })
} else {
(0, self.block_size)
};
@@ -316,6 +511,7 @@ mod tests {
disk::error::DiskError,
erasure_coding::{BitrotReader, BitrotWriter},
};
use bytes::Bytes;
use rustfs_utils::HashAlgorithm;
use std::io::Cursor;
@@ -456,4 +652,47 @@ mod tests {
let reader_cursor = Cursor::new(buf);
BitrotReader::new(reader_cursor, shard_size, hash_algo.clone(), false)
}
async fn create_bitrot_reader_from_shard(
shard: Bytes,
shard_size: usize,
hash_algo: &HashAlgorithm,
) -> BitrotReader<Cursor<Vec<u8>>> {
let writer = Cursor::new(Vec::new());
let mut writer = BitrotWriter::new(writer, shard_size, hash_algo.clone());
writer.write(shard.as_ref()).await.unwrap();
let reader_cursor = Cursor::new(writer.into_inner().into_inner());
BitrotReader::new(reader_cursor, shard_size, hash_algo.clone(), false)
}
#[tokio::test]
async fn test_erasure_chunk_decoder_reconstructs_missing_data_shard_as_pooled_chunks() {
let erasure = Erasure::new(2, 1, 4);
let original = b"abcd";
let encoded = erasure.encode_data(original).unwrap();
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::None;
let readers = vec![
None,
Some(create_bitrot_reader_from_shard(encoded[1].clone(), shard_size, &hash_algo).await),
Some(create_bitrot_reader_from_shard(encoded[2].clone(), shard_size, &hash_algo).await),
];
let mut decoder = ErasureChunkDecoder::new(erasure, readers, 0, original.len(), original.len()).unwrap();
let first_batch = decoder.next_chunks().await.unwrap().unwrap();
assert!(
first_batch.iter().all(|chunk| matches!(chunk, IoChunk::Pooled(_))),
"reconstructed decoder should produce pooled chunks"
);
let collected = first_batch
.into_iter()
.flat_map(|chunk| chunk.as_bytes().to_vec())
.collect::<Vec<_>>();
assert_eq!(collected, original);
assert!(decoder.next_chunks().await.unwrap().is_none());
assert_eq!(decoder.written(), original.len());
assert!(decoder.finish_error().is_none());
}
}
+308 -57
View File
@@ -17,11 +17,12 @@ use crate::disk::error_reduce::count_errs;
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs};
use crate::erasure_coding::BitrotWriterWrapper;
use crate::erasure_coding::Erasure;
use crate::erasure_coding::erasure::{EncodeBlockBuffer, EncodedShardBlock, EncodedShardBufferPool};
use bytes::Bytes;
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use rustfs_rio::BlockReadable;
use std::sync::Arc;
use std::vec;
use tokio::io::AsyncRead;
use tokio::sync::mpsc;
use tracing::error;
@@ -32,6 +33,164 @@ pub(crate) struct MultiWriter<'a> {
errs: Vec<Option<Error>>,
}
pub(crate) struct BlockAssembler<R> {
reader: R,
block_buffer: EncodeBlockBuffer,
total_bytes: usize,
}
impl<R> BlockAssembler<R>
where
R: AsyncRead + BlockReadable + Send + Sync + Unpin + 'static,
{
pub(crate) fn new(reader: R, block_size: usize) -> Self {
Self {
reader,
block_buffer: EncodeBlockBuffer::new(block_size),
total_bytes: 0,
}
}
pub(crate) async fn next_block(&mut self) -> std::io::Result<Option<Vec<u8>>> {
match self.block_buffer.read_from_block(&mut self.reader).await {
Ok(n) if n > 0 => {
self.total_bytes += n;
Ok(Some(self.block_buffer.filled(n).to_vec()))
}
Ok(_) => Ok(None),
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
if let Some(inner) = e.get_ref()
&& rustfs_rio::is_checksum_mismatch(inner)
{
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()));
}
Ok(None)
}
Err(e) => Err(e),
}
}
pub(crate) fn total_bytes(&self) -> usize {
self.total_bytes
}
pub(crate) fn into_inner(self) -> R {
self.reader
}
}
#[derive(Clone)]
pub(crate) struct ErasureChunkEncoder {
erasure: Arc<Erasure>,
buffer_pool: EncodedShardBufferPool,
}
impl ErasureChunkEncoder {
pub(crate) async fn new(erasure: Arc<Erasure>) -> Self {
let reusable_capacity = erasure.shard_size() * erasure.total_shard_count();
Self {
erasure,
buffer_pool: EncodedShardBufferPool::with_prefill(reusable_capacity, 2).await,
}
}
pub(crate) async fn encode_block(&self, block: &[u8]) -> std::io::Result<EncodedShardBlock> {
let reusable_buffer = self.buffer_pool.acquire().await;
self.erasure.encode_data_block_with_buffer(block, reusable_buffer)
}
pub(crate) async fn release(&self, block: EncodedShardBlock) {
self.buffer_pool.release(block).await;
}
}
pub(crate) struct ErasureWritePipeline {
erasure: Arc<Erasure>,
write_quorum: usize,
}
impl ErasureWritePipeline {
pub(crate) fn new(erasure: Arc<Erasure>, write_quorum: usize) -> Self {
Self { erasure, write_quorum }
}
pub(crate) async fn run<R>(&self, reader: R, writers: &mut [Option<BitrotWriterWrapper>]) -> std::io::Result<(R, usize)>
where
R: AsyncRead + BlockReadable + Send + Sync + Unpin + 'static,
{
let (tx, mut rx) = mpsc::channel::<EncodedShardBlock>(8);
let producer = ErasureChunkEncoder::new(self.erasure.clone()).await;
let writer_pool = producer.clone();
let block_size = self.erasure.block_size;
let task = tokio::spawn(async move {
let mut assembler = BlockAssembler::new(reader, block_size);
while let Some(block) = assembler.next_block().await? {
let res = producer.encode_block(&block).await?;
if let Err(err) = tx.send(res).await {
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
}
let total = assembler.total_bytes();
Ok((assembler.into_inner(), total))
});
let mut writers = MultiWriter::new(writers, self.write_quorum);
let mut write_err = None;
while let Some(block) = rx.recv().await {
if block.is_empty() {
break;
}
let write_result = writers.write(&block).await;
writer_pool.release(block).await;
if let Err(err) = write_result {
write_err = Some(err);
break;
}
}
if let Some(err) = write_err {
task.abort();
let _ = task.await;
if let Err(shutdown_err) = writers.shutdown().await {
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
}
return Err(err);
}
let (reader, total) = task.await??;
writers.shutdown().await?;
Ok((reader, total))
}
}
pub(crate) trait ShardSource {
fn shard_count(&self) -> usize;
fn shard(&self, idx: usize) -> Bytes;
}
impl ShardSource for EncodedShardBlock {
fn shard_count(&self) -> usize {
self.shard_count()
}
fn shard(&self, idx: usize) -> Bytes {
self.shard(idx)
}
}
impl ShardSource for Vec<Bytes> {
fn shard_count(&self) -> usize {
self.len()
}
fn shard(&self, idx: usize) -> Bytes {
self[idx].clone()
}
}
impl<'a> MultiWriter<'a> {
pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize) -> Self {
let length = writers.len();
@@ -42,10 +201,10 @@ impl<'a> MultiWriter<'a> {
}
}
async fn write_shard(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>, shard: &Bytes) {
async fn write_shard(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>, shard: Bytes) {
match writer_opt {
Some(writer) => {
match writer.write(shard).await {
match writer.write(&shard).await {
Ok(n) => {
if n < shard.len() {
*err = Some(Error::ShortWrite);
@@ -65,16 +224,40 @@ impl<'a> MultiWriter<'a> {
}
}
pub async fn write(&mut self, data: Vec<Bytes>) -> std::io::Result<()> {
assert_eq!(data.len(), self.writers.len());
fn write_shard_inline(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>, shard: Bytes) {
match writer_opt {
Some(writer) => match writer.write_inline_sync(&shard) {
Ok(n) => {
if n < shard.len() {
*err = Some(Error::ShortWrite);
*writer_opt = None;
} else {
*err = None;
}
}
Err(e) => {
*err = Some(Error::from(e));
}
},
None => {
*err = Some(Error::DiskNotFound);
}
}
}
pub async fn write<T>(&mut self, data: &T) -> std::io::Result<()>
where
T: ShardSource,
{
assert_eq!(data.shard_count(), self.writers.len());
{
let mut futures = FuturesUnordered::new();
for ((writer_opt, err), shard) in self.writers.iter_mut().zip(self.errs.iter_mut()).zip(data.iter()) {
for (idx, (writer_opt, err)) in self.writers.iter_mut().zip(self.errs.iter_mut()).enumerate() {
if err.is_some() {
continue; // Skip if we already have an error for this writer
}
futures.push(Self::write_shard(writer_opt, err, shard));
futures.push(Self::write_shard(writer_opt, err, data.shard(idx)));
}
while let Some(()) = futures.next().await {}
}
@@ -112,6 +295,45 @@ impl<'a> MultiWriter<'a> {
)))
}
pub fn write_inline<T>(&mut self, data: &T) -> std::io::Result<()>
where
T: ShardSource,
{
assert_eq!(data.shard_count(), self.writers.len());
for (idx, (writer_opt, err)) in self.writers.iter_mut().zip(self.errs.iter_mut()).enumerate() {
if err.is_some() {
continue;
}
Self::write_shard_inline(writer_opt, err, data.shard(idx));
}
let nil_count = self.errs.iter().filter(|&e| e.is_none()).count();
if nil_count >= self.write_quorum {
return Ok(());
}
if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
return Err(std::io::Error::other(format!(
"Failed to write inline data: {} (offline-disks={}/{})",
write_err,
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len()
)));
}
Err(std::io::Error::other(format!(
"Failed to write inline data: (offline-disks={}/{}): {}",
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len(),
self.errs
.iter()
.map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
.collect::<Vec<_>>()
.join(", ")
)))
}
async fn shutdown_writer(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>) {
match writer_opt {
Some(writer) => match writer.shutdown().await {
@@ -129,6 +351,23 @@ impl<'a> MultiWriter<'a> {
}
}
fn shutdown_writer_inline(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>) {
match writer_opt {
Some(writer) => match writer.shutdown_inline_sync() {
Ok(()) => {
*err = None;
}
Err(e) => {
*err = Some(Error::from(e));
*writer_opt = None;
}
},
None => {
*err = Some(Error::DiskNotFound);
}
}
}
pub async fn shutdown(&mut self) -> std::io::Result<()> {
{
let mut futures = FuturesUnordered::new();
@@ -173,66 +412,53 @@ impl<'a> MultiWriter<'a> {
.join(", ")
)))
}
pub fn shutdown_inline(&mut self) -> std::io::Result<()> {
for (writer_opt, err) in self.writers.iter_mut().zip(self.errs.iter_mut()) {
if err.is_some() {
continue;
}
Self::shutdown_writer_inline(writer_opt, err);
}
let nil_count = self.errs.iter().filter(|&e| e.is_none()).count();
if nil_count >= self.write_quorum {
return Ok(());
}
if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
return Err(std::io::Error::other(format!(
"Failed to shutdown inline writers: {} (offline-disks={}/{})",
write_err,
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len()
)));
}
Err(std::io::Error::other(format!(
"Failed to shutdown inline writers: (offline-disks={}/{}): {}",
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len(),
self.errs
.iter()
.map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
.collect::<Vec<_>>()
.join(", ")
)))
}
}
impl Erasure {
pub async fn encode<R>(
self: Arc<Self>,
mut reader: R,
reader: R,
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin + 'static,
R: AsyncRead + BlockReadable + Send + Sync + Unpin + 'static,
{
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
let task = tokio::spawn(async move {
let block_size = self.block_size;
let mut total = 0;
let mut buf = vec![0u8; block_size];
loop {
match rustfs_utils::read_full(&mut reader, &mut buf).await {
Ok(n) if n > 0 => {
total += n;
let res = self.encode_data(&buf[..n])?;
if let Err(err) = tx.send(res).await {
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
}
Ok(_) => {
break;
}
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
// Check if the inner error is a checksum mismatch - if so, propagate it
if let Some(inner) = e.get_ref()
&& rustfs_rio::is_checksum_mismatch(inner)
{
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()));
}
break;
}
Err(e) => {
return Err(e);
}
}
}
Ok((reader, total))
});
let mut writers = MultiWriter::new(writers, quorum);
while let Some(block) = rx.recv().await {
if block.is_empty() {
break;
}
writers.write(block).await?;
}
let (reader, total) = task.await??;
writers.shutdown().await?;
Ok((reader, total))
ErasureWritePipeline::new(self, quorum).run(reader, writers).await
}
}
@@ -241,6 +467,7 @@ mod tests {
use super::*;
use crate::erasure_coding::{BitrotWriterWrapper, CustomWriter};
use rustfs_utils::HashAlgorithm;
use std::io::Cursor;
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
@@ -296,4 +523,28 @@ mod tests {
assert_eq!(written, b"small payload".len());
assert!(!committed.lock().unwrap().is_empty());
}
#[tokio::test]
async fn block_assembler_splits_input_into_erasure_blocks() {
let reader = tokio::io::BufReader::new(Cursor::new(b"abcdefghijkl".to_vec()));
let mut assembler = BlockAssembler::new(reader, 4);
assert_eq!(assembler.next_block().await.unwrap(), Some(b"abcd".to_vec()));
assert_eq!(assembler.next_block().await.unwrap(), Some(b"efgh".to_vec()));
assert_eq!(assembler.next_block().await.unwrap(), Some(b"ijkl".to_vec()));
assert_eq!(assembler.next_block().await.unwrap(), None);
assert_eq!(assembler.total_bytes(), 12);
}
#[tokio::test]
async fn erasure_chunk_encoder_produces_full_shard_block() {
let erasure = Arc::new(Erasure::new(2, 1, 4));
let encoder = ErasureChunkEncoder::new(erasure.clone()).await;
let block = encoder.encode_block(b"abcd").await.unwrap();
assert_eq!(block.shard_count(), 3);
assert_eq!(block.shard(0).len(), erasure.shard_size());
encoder.release(block).await;
}
}
+254 -28
View File
@@ -19,12 +19,131 @@
use bytes::{Bytes, BytesMut};
use reed_solomon_erasure::galois_8::ReedSolomon;
use reed_solomon_simd;
use rustfs_rio::BlockReadable;
use smallvec::SmallVec;
use std::io;
use std::sync::Arc;
use tokio::io::AsyncRead;
use tokio::sync::Mutex;
use tracing::warn;
use uuid::Uuid;
pub(crate) struct EncodeBlockBuffer {
buf: Vec<u8>,
}
impl EncodeBlockBuffer {
pub(crate) fn new(block_size: usize) -> Self {
Self {
buf: vec![0u8; block_size],
}
}
pub(crate) async fn read_from<R>(&mut self, reader: &mut R) -> io::Result<usize>
where
R: AsyncRead + Send + Sync + Unpin,
{
rustfs_utils::read_full(&mut *reader, &mut self.buf).await
}
pub(crate) async fn read_from_block<R>(&mut self, reader: &mut R) -> io::Result<usize>
where
R: BlockReadable + Send + Sync + Unpin,
{
reader.read_block(&mut self.buf).await
}
pub(crate) fn filled(&self, len: usize) -> &[u8] {
&self.buf[..len]
}
}
pub struct EncodedShardBlock {
data: Bytes,
shard_size: usize,
shard_count: usize,
}
impl EncodedShardBlock {
pub(crate) fn new(data: Bytes, shard_size: usize, shard_count: usize) -> Self {
Self {
data,
shard_size,
shard_count,
}
}
pub fn shard_count(&self) -> usize {
self.shard_count
}
pub fn len(&self) -> usize {
self.shard_count
}
pub fn is_empty(&self) -> bool {
self.shard_count == 0
}
pub fn shard(&self, idx: usize) -> Bytes {
let start = idx * self.shard_size;
let end = start + self.shard_size;
self.data.slice(start..end)
}
pub fn iter(&self) -> impl Iterator<Item = Bytes> + '_ {
(0..self.shard_count).map(|idx| self.shard(idx))
}
pub fn into_vec(self) -> Vec<Bytes> {
(0..self.shard_count).map(|idx| self.shard(idx)).collect()
}
pub fn into_reusable_buffer(self) -> BytesMut {
match self.data.try_into_mut() {
Ok(mut buf) => {
buf.clear();
buf
}
Err(data) => BytesMut::with_capacity(data.len()),
}
}
}
#[derive(Clone)]
pub(crate) struct EncodedShardBufferPool {
capacity: usize,
free: Arc<Mutex<Vec<BytesMut>>>,
}
impl EncodedShardBufferPool {
pub(crate) async fn with_prefill(capacity: usize, initial: usize) -> Self {
let mut free = Vec::with_capacity(initial);
for _ in 0..initial {
free.push(BytesMut::with_capacity(capacity));
}
Self {
capacity,
free: Arc::new(Mutex::new(free)),
}
}
pub(crate) async fn acquire(&self) -> BytesMut {
let mut free = self.free.lock().await;
free.pop().unwrap_or_else(|| BytesMut::with_capacity(self.capacity))
}
pub(crate) async fn release(&self, block: EncodedShardBlock) {
let mut free = self.free.lock().await;
let mut buf = block.into_reusable_buffer();
if buf.capacity() < self.capacity {
buf.reserve(self.capacity - buf.capacity());
}
free.push(buf);
}
}
/// Legacy calc_shard_size formula: (block_size.div_ceil(data_shards) + 1) & !1
/// Matches main branch and filemeta::ErasureInfo for old-version files.
pub fn calc_shard_size_legacy(block_size: usize, data_shards: usize) -> usize {
@@ -351,6 +470,25 @@ impl Erasure {
/// A vector of encoded shards as `Bytes`.
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
Ok(self.encode_data_block(data)?.into_vec())
}
/// Encode one logical block into an `EncodedShardBlock` using a caller-provided backing buffer.
///
/// This is the explicit reuse-oriented variant for non-hot paths that want to
/// thread a reusable `BytesMut` across multiple encode calls.
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
pub fn encode_data_with_buffer(&self, data: &[u8], data_buffer: BytesMut) -> io::Result<EncodedShardBlock> {
self.encode_data_block_with_buffer(data, data_buffer)
}
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
pub(crate) fn encode_data_block(&self, data: &[u8]) -> io::Result<EncodedShardBlock> {
self.encode_data_block_with_buffer(data, BytesMut::with_capacity(self.shard_size() * self.total_shard_count()))
}
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
pub(crate) fn encode_data_block_with_buffer(&self, data: &[u8], mut data_buffer: BytesMut) -> io::Result<EncodedShardBlock> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
} else {
@@ -359,7 +497,10 @@ impl Erasure {
let per_shard_size = shard_size_fn(data.len(), self.data_shards);
let need_total_size = per_shard_size * self.total_shard_count();
let mut data_buffer = BytesMut::with_capacity(need_total_size);
data_buffer.clear();
if data_buffer.capacity() < need_total_size {
data_buffer.reserve(need_total_size - data_buffer.capacity());
}
data_buffer.extend_from_slice(data);
data_buffer.resize(need_total_size, 0u8);
@@ -382,14 +523,7 @@ impl Erasure {
}
// Zero-copy split, all shards reference data_buffer
let mut data_buffer = data_buffer.freeze();
let mut shards = Vec::with_capacity(self.total_shard_count());
for _ in 0..self.total_shard_count() {
let shard = data_buffer.split_to(per_shard_size);
shards.push(shard);
}
Ok(shards)
Ok(EncodedShardBlock::new(data_buffer.freeze(), per_shard_size, self.total_shard_count()))
}
/// Decode and reconstruct missing shards in-place.
@@ -478,8 +612,8 @@ impl Erasure {
///
/// # Arguments
/// * `reader` - An async reader implementing AsyncRead + Send + Sync + Unpin
/// * `mut on_block` - Async callback that receives encoded blocks and returns a Result
/// * `F` - Callback type: FnMut(Result<Vec<Bytes>, std::io::Error>) -> Future<Output=Result<(), E>> + Send
/// * `mut on_block` - Async callback that receives encoded blocks and returns the block for reuse
/// * `F` - Callback type: FnMut(Result<EncodedShardBlock, std::io::Error>) -> Future<Output=Result<Option<EncodedShardBlock>, E>> + Send
/// * `Fut` - Future type returned by the callback
/// * `E` - Error type returned by the callback
/// * `R` - Reader type implementing AsyncRead + Send + Sync + Unpin
@@ -489,26 +623,31 @@ impl Erasure {
///
/// # Errors
/// Returns error if reading from reader fails or if callback returns error
pub async fn encode_stream_callback_async<F, Fut, E, R>(
pub(crate) async fn encode_stream_callback_async<F, Fut, E, R>(
self: std::sync::Arc<Self>,
reader: &mut R,
mut on_block: F,
) -> Result<usize, E>
where
R: AsyncRead + Send + Sync + Unpin,
F: FnMut(std::io::Result<Vec<Bytes>>) -> Fut + Send,
Fut: std::future::Future<Output = Result<(), E>> + Send,
F: FnMut(std::io::Result<EncodedShardBlock>) -> Fut + Send,
Fut: std::future::Future<Output = Result<Option<EncodedShardBlock>, E>> + Send,
{
let block_size = self.block_size;
let mut total = 0;
let mut block_buffer = EncodeBlockBuffer::new(block_size);
let reusable_capacity = self.shard_size() * self.total_shard_count();
let buffer_pool = EncodedShardBufferPool::with_prefill(reusable_capacity, 1).await;
loop {
let mut buf = vec![0u8; block_size];
match rustfs_utils::read_full(&mut *reader, &mut buf).await {
match block_buffer.read_from(&mut *reader).await {
Ok(n) if n > 0 => {
warn!("encode_stream_callback_async read n={}", n);
total += n;
let res = self.encode_data(&buf[..n]);
on_block(res).await?
let reusable_buffer = buffer_pool.acquire().await;
let res = self.encode_data_block_with_buffer(block_buffer.filled(n), reusable_buffer);
if let Some(block) = on_block(res).await? {
buffer_pool.release(block).await;
}
}
Ok(_) => {
warn!("encode_stream_callback_async read unexpected ok");
@@ -520,11 +659,10 @@ impl Erasure {
}
Err(e) => {
warn!("encode_stream_callback_async read error={:?}", e);
on_block(Err(e)).await?;
let _ = on_block(Err(e)).await?;
break;
}
}
buf.clear();
}
Ok(total)
}
@@ -747,8 +885,8 @@ mod tests {
let tx = tx.clone();
async move {
let shards = res.unwrap();
tx.send(shards).await.unwrap();
Ok(())
tx.send(shards.iter().collect()).await.unwrap();
Ok(Some(shards))
}
})
.await
@@ -760,6 +898,36 @@ mod tests {
assert_eq!(collected_shards.len(), data_shards + parity_shards);
}
#[test]
fn test_encode_data_with_buffer_supports_explicit_reuse() {
let erasure = Erasure::new(4, 2, 1024);
let reusable_capacity = erasure.shard_size() * erasure.total_shard_count();
let first_data = b"explicit reusable buffer path".repeat(32);
let first_block = erasure
.encode_data_with_buffer(&first_data, BytesMut::with_capacity(reusable_capacity))
.expect("first encode should succeed");
let reusable_buffer = first_block.into_reusable_buffer();
assert!(reusable_buffer.capacity() >= reusable_capacity);
let second_data = b"second encode through same reusable buffer".repeat(24);
let second_block = erasure
.encode_data_with_buffer(&second_data, reusable_buffer)
.expect("second encode should succeed");
let mut shards_opt: Vec<Option<Vec<u8>>> = second_block.iter().map(|shard| Some(shard.to_vec())).collect();
shards_opt[1] = None;
shards_opt[5] = None;
erasure.decode_data(&mut shards_opt).expect("decode should succeed");
let mut recovered = Vec::new();
for shard in shards_opt.iter().take(erasure.data_shards) {
recovered.extend_from_slice(shard.as_ref().expect("data shard should exist after decode"));
}
recovered.truncate(second_data.len());
assert_eq!(&recovered, &second_data);
}
#[tokio::test]
async fn test_encode_stream_callback_async_channel_decode() {
use std::io::Cursor;
@@ -786,8 +954,8 @@ mod tests {
let tx = tx.clone();
async move {
let shards = res.unwrap();
tx.send(shards).await.unwrap();
Ok(())
tx.send(shards.iter().collect()).await.unwrap();
Ok(Some(shards))
}
})
.await
@@ -800,8 +968,8 @@ mod tests {
// Test decode using the old API that operates in-place
let mut decode_input: Vec<Option<Vec<u8>>> = vec![None; data_shards + parity_shards];
for i in 0..data_shards {
decode_input[i] = Some(shards[i].to_vec());
for (i, shard) in shards.iter().enumerate().take(data_shards) {
decode_input[i] = Some(shard.to_vec());
}
erasure.decode_data(&mut decode_input).unwrap();
@@ -1198,8 +1366,8 @@ mod tests {
let tx = tx.clone();
async move {
let shards = res.unwrap();
tx.send(shards).await.unwrap();
Ok(())
tx.send(shards.iter().collect()).await.unwrap();
Ok(Some(shards))
}
})
.await
@@ -1233,5 +1401,63 @@ mod tests {
recovered.truncate(data_clone.len());
assert_eq!(&recovered, &data_clone);
}
#[tokio::test]
#[ignore]
async fn stress_simd_stream_callback_reuses_backing_buffers_across_many_blocks() {
use std::io::Cursor;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::sync::Mutex;
let data_shards = 4;
let parity_shards = 2;
let block_size = 1024;
let erasure = Arc::new(Erasure::new(data_shards, parity_shards, block_size));
let sample =
b"SIMD stress callback test payload that intentionally spans many blocks to exercise reusable backing buffers.";
let data = sample.repeat((4 * 1024 * 1024 / sample.len()).max(1));
let data_clone = data.clone();
let mut reader = Cursor::new(data);
let recovered = Arc::new(Mutex::new(Vec::with_capacity(data_clone.len())));
let block_count = Arc::new(AtomicUsize::new(0));
let erasure_for_callback = erasure.clone();
let recovered_for_callback = recovered.clone();
let block_count_for_callback = block_count.clone();
erasure
.clone()
.encode_stream_callback_async::<_, _, (), _>(&mut reader, move |res| {
let erasure_for_callback = erasure_for_callback.clone();
let recovered_for_callback = recovered_for_callback.clone();
let block_count_for_callback = block_count_for_callback.clone();
async move {
let shards = res.unwrap();
block_count_for_callback.fetch_add(1, Ordering::Relaxed);
let mut shards_opt: Vec<Option<Vec<u8>>> = shards.iter().map(|b| Some(b.to_vec())).collect();
shards_opt[1] = None;
shards_opt[5] = None;
erasure_for_callback.decode_data(&mut shards_opt).unwrap();
let mut recovered = recovered_for_callback.lock().await;
for shard in shards_opt.iter().take(data_shards) {
recovered.extend_from_slice(shard.as_ref().unwrap());
}
Ok(Some(shards))
}
})
.await
.unwrap();
assert!(block_count.load(Ordering::Relaxed) > 1024);
let mut recovered = recovered.lock().await;
recovered.truncate(data_clone.len());
assert_eq!(&*recovered, &data_clone);
}
}
}
+1 -1
View File
@@ -77,7 +77,7 @@ impl super::Erasure {
let available_writers = writers.iter().filter(|w| w.is_some()).count();
let write_quorum = available_writers.max(1); // At least 1 writer must succeed
let mut writers = MultiWriter::new(writers, write_quorum);
writers.write(shards).await?;
writers.write(&shards).await?;
}
Ok(())
+30 -1
View File
@@ -30,8 +30,10 @@ use crate::{
};
use bytes::Bytes;
use futures::lock::Mutex;
use futures_util::StreamExt;
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_core::{BoxChunkStream, IoChunk};
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_protos::proto_gen::node_service::{
CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
@@ -40,7 +42,7 @@ use rustfs_protos::proto_gen::node_service::{
RenameFileRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
node_service_client::NodeServiceClient,
};
use rustfs_rio::{HttpReader, HttpWriter};
use rustfs_rio::{HttpReader, HttpWriter, open_http_byte_stream};
use serde::{Serialize, de::DeserializeOwned};
use std::{
io::Cursor,
@@ -1071,6 +1073,33 @@ impl DiskAPI for RemoteDisk {
Ok(Bytes::from(buffer))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<BoxChunkStream> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let disk = self.disk_ref().await;
let url = format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
self.endpoint.grid_host(),
urlencoding::encode(&disk),
urlencoding::encode(volume),
urlencoding::encode(path),
offset,
length
);
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
build_auth_headers(&url, &Method::GET, &mut headers);
let stream = open_http_byte_stream(url, Method::GET, headers, None)
.await?
.map(|result| result.map(IoChunk::Shared));
Ok(Box::pin(stream))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
info!("append_file {}/{}", volume, path);
+183 -39
View File
@@ -52,9 +52,9 @@ use crate::{
event_notification::{EventArgs, send_event},
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure},
store_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader,
HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectsV2Info, ListOperations, MakeBucketOptions, MultipartInfo,
MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations, PartInfo, PutObjReader, StorageAPI,
BucketInfo, BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, DeleteBucketOptions, DeletedObject,
GetObjectReader, HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectsV2Info, ListOperations, MakeBucketOptions,
MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations, PartInfo, StorageAPI,
},
store_init::load_format_erasure,
};
@@ -118,6 +118,66 @@ use tokio::{
time::{interval, timeout},
};
use tokio_util::sync::CancellationToken;
const ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES: &str = "RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES";
const ENV_RUSTFS_PUT_FORCE_DISABLE_INLINE: &str = "RUSTFS_PUT_FORCE_DISABLE_INLINE";
const SLOW_PUT_STORAGE_PHASE_DEBUG_THRESHOLD_MS: u64 = 100;
const SLOW_PUT_STORAGE_PHASE_WARN_THRESHOLD_MS: u64 = 1_000;
const SLOW_PUT_STORAGE_PHASE_ERROR_THRESHOLD_MS: u64 = 5_000;
fn env_flag_enabled(name: &str) -> bool {
rustfs_utils::get_env_bool(name, false)
}
fn env_non_negative_usize(name: &str) -> Option<usize> {
rustfs_utils::get_env_opt_usize(name)
}
fn resolved_put_inline_buffer_enabled(object_size: i64, inline_by_topology: bool) -> bool {
if !inline_by_topology || object_size < 0 {
return false;
}
if env_flag_enabled(ENV_RUSTFS_PUT_FORCE_DISABLE_INLINE) {
return false;
}
env_non_negative_usize(ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES)
.map(|value| usize::try_from(object_size).is_ok_and(|size| size <= value))
.unwrap_or(inline_by_topology)
}
fn log_put_storage_phase(
bucket: &str,
object: &str,
phase: &str,
elapsed: Duration,
object_size: i64,
inline_selected: bool,
write_quorum: usize,
) {
let duration_ms = elapsed.as_millis() as u64;
if duration_ms < SLOW_PUT_STORAGE_PHASE_DEBUG_THRESHOLD_MS {
return;
}
if duration_ms >= SLOW_PUT_STORAGE_PHASE_ERROR_THRESHOLD_MS {
error!(
phase,
duration_ms, object_size, inline_selected, write_quorum, bucket, object, "PUT storage phase is critically slow"
);
} else if duration_ms >= SLOW_PUT_STORAGE_PHASE_WARN_THRESHOLD_MS {
warn!(
phase,
duration_ms, object_size, inline_selected, write_quorum, bucket, object, "PUT storage phase is slow"
);
} else {
debug!(
phase,
duration_ms, object_size, inline_selected, write_quorum, bucket, object, "PUT storage phase exceeded debug threshold"
);
}
}
use tracing::error;
use tracing::{debug, info, warn};
use uuid::Uuid;
@@ -151,6 +211,9 @@ mod read;
mod replication;
mod write;
#[doc(hidden)]
pub use read::collect_direct_data_shard_chunks_for_benchmark;
/// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
/// Defaults to 30 seconds if not set or invalid
pub fn get_lock_acquire_timeout() -> Duration {
@@ -692,7 +755,13 @@ impl ObjectIO for SetDisks {
}
#[tracing::instrument(skip(self, data,))]
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
async fn put_object(
&self,
bucket: &str,
object: &str,
data: &mut ChunkNativePutData,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
let disks = self.get_disks_internal().await;
let mut object_lock_guard = None;
@@ -774,13 +843,18 @@ impl ObjectIO for SetDisks {
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let is_inline_buffer = {
if let Some(sc) = GLOBAL_STORAGE_CLASS.get() {
let inline_by_topology = if let Some(sc) = GLOBAL_STORAGE_CLASS.get() {
sc.should_inline(erasure.shard_file_size(data.size()), opts.versioned)
} else {
false
}
};
resolved_put_inline_buffer_enabled(data.size(), inline_by_topology)
};
if is_inline_buffer {
rustfs_io_metrics::record_put_inline_selected(data.size(), opts.versioned);
}
let writer_setup_start = Instant::now();
let mut writers = Vec::with_capacity(shuffle_disks.len());
let mut errors = Vec::with_capacity(shuffle_disks.len());
for disk_op in shuffle_disks.iter() {
@@ -814,6 +888,15 @@ impl ObjectIO for SetDisks {
writers.push(None);
}
}
log_put_storage_phase(
bucket,
object,
"writer_setup",
writer_setup_start.elapsed(),
data.size(),
is_inline_buffer,
write_quorum,
);
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
if nil_count < write_quorum {
@@ -825,23 +908,33 @@ impl ObjectIO for SetDisks {
return Err(Error::other(format!("not enough disks to write: {errors:?}")));
}
let stream = mem::replace(
&mut data.stream,
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
);
let (reader, w_size) = match Arc::new(erasure).encode(stream, &mut writers, write_quorum).await {
Ok((r, w)) => (r, w),
let object_size = data.size();
let encode_write_start = Instant::now();
let w_size = match Self::write_chunk_native_put_data(data, Arc::new(erasure), &mut writers, write_quorum).await {
Ok(written) => written,
Err(e) => {
log_put_storage_phase(
bucket,
object,
"encode_write",
encode_write_start.elapsed(),
object_size,
is_inline_buffer,
write_quorum,
);
error!("encode err {:?}", e);
return Err(e.into());
}
}; // TODO: delete temporary directory on error
let _ = mem::replace(&mut data.stream, reader);
// if let Err(err) = close_bitrot_writers(&mut writers).await {
// error!("close_bitrot_writers err {:?}", err);
// }
log_put_storage_phase(
bucket,
object,
"encode_write",
encode_write_start.elapsed(),
data.size(),
is_inline_buffer,
write_quorum,
);
if (w_size as i64) < data.size() {
warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size());
@@ -856,11 +949,11 @@ impl ObjectIO for SetDisks {
insert_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE, w_size.to_string());
}
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
let index_op = data.index_bytes();
//TODO: userDefined
let etag = data.stream.try_resolve_etag().unwrap_or_default();
let etag = data.resolve_etag().unwrap_or_default();
user_defined.insert("etag".to_owned(), etag.clone());
@@ -877,9 +970,9 @@ impl ObjectIO for SetDisks {
}
if fi.checksum.is_none()
&& let Some(content_hash) = data.as_hash_reader().content_hash()
&& let Some(content_hash) = data.content_hash_bytes()?
{
fi.checksum = Some(content_hash.to_bytes(&[]));
fi.checksum = Some(content_hash);
}
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
@@ -917,6 +1010,7 @@ impl ObjectIO for SetDisks {
drop(writers); // drop writers to close all files, this is to prevent FileAccessDenied errors when renaming data
let post_write_lock_start = Instant::now();
if !opts.no_lock && object_lock_guard.is_none() {
let ns_lock = self.new_ns_lock(bucket, object).await?;
object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
@@ -926,7 +1020,17 @@ impl ObjectIO for SetDisks {
))
})?);
}
log_put_storage_phase(
bucket,
object,
"post_write_lock",
post_write_lock_start.elapsed(),
data.size(),
is_inline_buffer,
write_quorum,
);
let finalize_start = Instant::now();
let (online_disks, _, op_old_dir) = Self::rename_data(
&shuffle_disks,
RUSTFS_META_TMP_BUCKET,
@@ -936,7 +1040,18 @@ impl ObjectIO for SetDisks {
object,
write_quorum,
)
.await?;
.await
.inspect_err(|_| {
log_put_storage_phase(
bucket,
object,
"finalize",
finalize_start.elapsed(),
data.size(),
is_inline_buffer,
write_quorum,
);
})?;
if let Some(old_dir) = op_old_dir {
self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum)
@@ -946,6 +1061,15 @@ impl ObjectIO for SetDisks {
drop(object_lock_guard); // drop object lock guard to release the lock
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
log_put_storage_phase(
bucket,
object,
"finalize",
finalize_start.elapsed(),
data.size(),
is_inline_buffer,
write_quorum,
);
for (i, op_disk) in online_disks.iter().enumerate() {
if let Some(disk) = op_disk
@@ -1864,6 +1988,9 @@ impl ObjectOperations for SetDisks {
if let Some(ref version_id) = opts.version_id {
fi.version_id = Uuid::parse_str(version_id).ok();
}
if let Some(checksum) = &opts.resolved_checksum {
fi.checksum = Some(checksum.clone());
}
self.update_object_meta(bucket, object, fi.clone(), &online_disks)
.await
@@ -2090,7 +2217,7 @@ impl ObjectOperations for SetDisks {
let gr = gr.unwrap();
let reader = BufReader::new(gr.stream);
let hash_reader = HashReader::from_stream(reader, gr.object_info.size, gr.object_info.size, None, None, false)?;
let mut p_reader = PutObjReader::new(hash_reader);
let mut p_reader = ChunkNativePutData::new(hash_reader);
return match self_.clone().put_object(bucket, object, &mut p_reader, &ropts).await {
Ok(restored_info) => {
send_event(EventArgs {
@@ -2158,7 +2285,7 @@ impl ObjectOperations for SetDisks {
};
let reader = BufReader::new(gr.stream);
let hash_reader = HashReader::from_stream(reader, part_info.actual_size, part_info.actual_size, None, None, false)?;
let mut p_reader = PutObjReader::new(hash_reader);
let mut p_reader = ChunkNativePutData::new(hash_reader);
let p_info = self_
.clone()
.put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default())
@@ -2382,7 +2509,7 @@ impl MultipartOperations for SetDisks {
object: &str,
upload_id: &str,
part_id: usize,
data: &mut PutObjReader,
data: &mut ChunkNativePutData,
opts: &ObjectOptions,
) -> Result<PartInfo> {
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
@@ -2394,9 +2521,8 @@ impl MultipartOperations for SetDisks {
if let Some(checksum) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
&& !checksum.is_empty()
&& data
.as_hash_reader()
.content_crc_type()
.is_none_or(|v| v.to_string() != *checksum)
.is_none_or(|v: rustfs_rio::ChecksumType| v.to_string() != *checksum)
{
return Err(Error::other(format!("checksum mismatch: {checksum}")));
}
@@ -2461,14 +2587,7 @@ impl MultipartOperations for SetDisks {
return Err(Error::other(format!("not enough disks to write: {errors:?}")));
}
let stream = mem::replace(
&mut data.stream,
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
);
let (reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: delete temporary directory on error
let _ = mem::replace(&mut data.stream, reader);
let w_size = Self::write_chunk_native_put_data(data, Arc::new(erasure), &mut writers, write_quorum).await?; // TODO: delete temporary directory on error
if (w_size as i64) < data.size() {
warn!("put_object_part write size < data.size(), w_size={}, data.size={}", w_size, data.size());
@@ -2479,9 +2598,9 @@ impl MultipartOperations for SetDisks {
)));
}
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
let index_op = data.index_bytes();
let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
let mut etag = data.resolve_etag().unwrap_or_default();
if let Some(ref tag) = opts.preserve_etag {
etag = tag.clone();
@@ -2495,7 +2614,7 @@ impl MultipartOperations for SetDisks {
}
}
let checksums = data.as_hash_reader().content_crc();
let checksums = data.content_crc();
let part_info = ObjectPartInfo {
etag: etag.clone(),
@@ -4212,6 +4331,31 @@ mod tests {
}
}
#[test]
#[serial]
fn resolved_put_inline_buffer_enabled_honors_disable_env() {
temp_env::with_var(ENV_RUSTFS_PUT_FORCE_DISABLE_INLINE, Some("true"), || {
assert!(!resolved_put_inline_buffer_enabled(4096, true));
});
}
#[test]
#[serial]
fn resolved_put_inline_buffer_enabled_honors_max_bytes_override() {
temp_env::with_var(ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES, Some("4096"), || {
assert!(resolved_put_inline_buffer_enabled(4096, true));
assert!(!resolved_put_inline_buffer_enabled(4097, true));
});
}
#[test]
#[serial]
fn resolved_put_inline_buffer_enabled_ignores_invalid_override() {
temp_env::with_var(ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES, Some("invalid"), || {
assert!(resolved_put_inline_buffer_enabled(4096, true));
});
}
async fn current_setup_type() -> SetupType {
if is_dist_erasure().await {
SetupType::DistErasure
File diff suppressed because it is too large Load Diff
+132
View File
@@ -13,12 +13,87 @@
// limitations under the License.
use super::*;
use crate::store_api::ChunkNativePutData;
impl SetDisks {
fn all_inline_bitrot_writers(writers: &[Option<crate::erasure_coding::BitrotWriterWrapper>]) -> bool {
writers.iter().all(|writer| {
writer
.as_ref()
.is_some_and(crate::erasure_coding::BitrotWriterWrapper::is_inline_buffer)
})
}
async fn write_chunk_native_put_data_inline(
data: &mut ChunkNativePutData,
erasure: Arc<erasure_coding::Erasure>,
writers: &mut [Option<crate::erasure_coding::BitrotWriterWrapper>],
write_quorum: usize,
) -> std::io::Result<usize> {
let stream = data.take_stream()?;
let mut assembler = erasure_coding::encode::BlockAssembler::new(stream, erasure.block_size);
let encoder = erasure_coding::encode::ErasureChunkEncoder::new(erasure).await;
let mut writer_group = erasure_coding::encode::MultiWriter::new(writers, write_quorum);
loop {
let block = match assembler.next_block().await {
Ok(Some(block)) => block,
Ok(None) => break,
Err(err) => {
data.restore_stream(assembler.into_inner());
return Err(err);
}
};
let encoded = match encoder.encode_block(&block).await {
Ok(encoded) => encoded,
Err(err) => {
data.restore_stream(assembler.into_inner());
return Err(err);
}
};
if let Err(err) = writer_group.write_inline(&encoded) {
encoder.release(encoded).await;
data.restore_stream(assembler.into_inner());
return Err(err);
}
encoder.release(encoded).await;
}
let total_bytes = assembler.total_bytes();
let stream = assembler.into_inner();
if let Err(err) = writer_group.shutdown_inline() {
data.restore_stream(stream);
return Err(err);
}
data.restore_stream(stream);
Ok(total_bytes)
}
pub(super) fn default_read_quorum(&self) -> usize {
self.set_drive_count - self.default_parity_count
}
pub(super) async fn write_chunk_native_put_data(
data: &mut ChunkNativePutData,
erasure: Arc<erasure_coding::Erasure>,
writers: &mut [Option<crate::erasure_coding::BitrotWriterWrapper>],
write_quorum: usize,
) -> std::io::Result<usize> {
if Self::all_inline_bitrot_writers(writers) {
return Self::write_chunk_native_put_data_inline(data, erasure, writers, write_quorum).await;
}
let stream = data.take_stream()?;
let (stream, written) = erasure_coding::encode::ErasureWritePipeline::new(erasure, write_quorum)
.run(stream, writers)
.await?;
data.restore_stream(stream);
Ok(written)
}
pub(super) fn default_write_quorum(&self) -> usize {
let mut data_count = self.set_drive_count - self.default_parity_count;
if data_count == self.default_parity_count {
@@ -627,3 +702,60 @@ impl SetDisks {
None
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::erasure_coding::{BitrotWriterWrapper, CustomWriter};
use crate::store_api::ChunkNativePutData;
#[tokio::test]
async fn write_chunk_native_put_data_restores_reader_state_after_encoding() {
let payload = b"chunk-native-put-payload".repeat(8);
let erasure = Arc::new(erasure_coding::Erasure::new(2, 1, 8));
let mut reader = ChunkNativePutData::from_vec(payload.clone());
let mut writers: Vec<Option<BitrotWriterWrapper>> = (0..erasure.total_shard_count())
.map(|_| {
Some(BitrotWriterWrapper::new(
CustomWriter::new_inline_buffer(),
erasure.shard_size(),
HashAlgorithm::HighwayHash256S,
))
})
.collect();
let written = SetDisks::write_chunk_native_put_data(&mut reader, erasure.clone(), &mut writers, 2)
.await
.unwrap();
assert_eq!(written, payload.len());
assert_eq!(reader.size(), payload.len() as i64);
assert_eq!(reader.actual_size(), payload.len() as i64);
assert!(
reader.resolve_etag().is_some(),
"restored reader should preserve computed etag state after chunk-native encode"
);
let inline_lengths: Vec<usize> = writers
.into_iter()
.map(|writer| writer.expect("writer").into_inline_data().expect("inline data").len())
.collect();
assert!(inline_lengths.iter().all(|len| *len > 0));
}
#[tokio::test]
async fn write_chunk_native_put_data_detects_inline_writer_set() {
let erasure = Arc::new(erasure_coding::Erasure::new(2, 1, 8));
let writers: Vec<Option<BitrotWriterWrapper>> = (0..erasure.total_shard_count())
.map(|_| {
Some(BitrotWriterWrapper::new(
CustomWriter::new_inline_buffer(),
erasure.shard_size(),
HashAlgorithm::HighwayHash256S,
))
})
.collect();
assert!(SetDisks::all_inline_bitrot_writers(&writers));
}
}
+26 -7
View File
@@ -15,7 +15,7 @@
use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result};
use crate::store_api::{ListPartsInfo, ObjectInfoOrErr, WalkOptions};
use crate::store_api::{GetObjectChunkResult, ListPartsInfo, ObjectInfoOrErr, WalkOptions};
use crate::{
disk::{
DiskAPI, DiskInfo, DiskOption, DiskStore,
@@ -28,10 +28,10 @@ use crate::{
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure},
set_disk::SetDisks,
store_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader,
HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations,
MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations,
ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI,
BucketInfo, BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, DeleteBucketOptions, DeletedObject,
GetObjectReader, HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info,
ListOperations, MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo,
ObjectOperations, ObjectOptions, ObjectToDelete, PartInfo, StorageAPI,
},
store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
};
@@ -287,6 +287,19 @@ impl Sets {
self.get_disks(self.get_hashed_set_index(key))
}
pub async fn get_object_chunks(
&self,
bucket: &str,
object: &str,
range: Option<HTTPRangeSpec>,
h: HeaderMap,
opts: &ObjectOptions,
) -> Result<GetObjectChunkResult> {
self.get_disks_by_key(object)
.get_object_chunks(bucket, object, range, h, opts)
.await
}
fn get_hashed_set_index(&self, input: &str) -> usize {
match self.distribution_algo {
DistributionAlgoVersion::V1 => crc_hash(input, self.disk_set.len()),
@@ -375,7 +388,13 @@ impl ObjectIO for Sets {
.await
}
#[tracing::instrument(level = "debug", skip(self, data))]
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
async fn put_object(
&self,
bucket: &str,
object: &str,
data: &mut ChunkNativePutData,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
self.get_disks_by_key(object).put_object(bucket, object, data, opts).await
}
}
@@ -688,7 +707,7 @@ impl MultipartOperations for Sets {
object: &str,
upload_id: &str,
part_id: usize,
data: &mut PutObjReader,
data: &mut ChunkNativePutData,
opts: &ObjectOptions,
) -> Result<PartInfo> {
self.get_disks_by_key(object)
+26 -5
View File
@@ -59,9 +59,10 @@ use crate::{
rpc::S3PeerSys,
sets::Sets,
store_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader,
HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations, MakeBucketOptions, MultipartOperations,
MultipartUploadResult, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI,
BucketInfo, BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, DeleteBucketOptions, DeletedObject,
GetObjectChunkResult, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations,
MakeBucketOptions, MultipartOperations, MultipartUploadResult, ObjectInfo, ObjectOperations, ObjectOptions,
ObjectToDelete, PartInfo, StorageAPI,
},
store_init,
};
@@ -259,11 +260,31 @@ impl ObjectIO for ECStore {
self.handle_get_object_reader(bucket, object, range, h, opts).await
}
#[instrument(level = "debug", skip(self, data))]
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
async fn put_object(
&self,
bucket: &str,
object: &str,
data: &mut ChunkNativePutData,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
enqueue_transition_after_write(self.handle_put_object(bucket, object, data, opts).await, LcEventSrc::S3PutObject).await
}
}
impl ECStore {
#[instrument(level = "debug", skip(self))]
pub async fn get_object_chunks(
&self,
bucket: &str,
object: &str,
range: Option<HTTPRangeSpec>,
h: HeaderMap,
opts: &ObjectOptions,
) -> Result<GetObjectChunkResult> {
self.handle_get_object_chunks(bucket, object, range, h, opts).await
}
}
lazy_static! {
static ref enableObjcetLockConfig: ObjectLockConfiguration = ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
@@ -495,7 +516,7 @@ impl MultipartOperations for ECStore {
object: &str,
upload_id: &str,
part_id: usize,
data: &mut PutObjReader,
data: &mut ChunkNativePutData,
opts: &ObjectOptions,
) -> Result<PartInfo> {
self.handle_put_object_part(bucket, object, upload_id, part_id, data, opts)
+1 -1
View File
@@ -176,7 +176,7 @@ impl ECStore {
object: &str,
upload_id: &str,
part_id: usize,
data: &mut PutObjReader,
data: &mut ChunkNativePutData,
opts: &ObjectOptions,
) -> Result<PartInfo> {
check_put_object_part_args(bucket, object, upload_id)?;
+30 -1
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use super::*;
use crate::store_api::GetObjectChunkResult;
fn select_data_movement_target_pool(
existing_pool_idx: Result<usize>,
@@ -213,12 +214,40 @@ impl ECStore {
.await
}
#[instrument(level = "debug", skip(self))]
pub(super) async fn handle_get_object_chunks(
&self,
bucket: &str,
object: &str,
range: Option<HTTPRangeSpec>,
h: HeaderMap,
opts: &ObjectOptions,
) -> Result<GetObjectChunkResult> {
check_get_obj_args(bucket, object)?;
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].get_object_chunks(bucket, object.as_str(), range, h, opts).await;
}
let mut opts = opts.clone();
opts.no_lock = true;
let (_, idx) = self
.get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
.await?;
self.pools[idx]
.get_object_chunks(bucket, object.as_str(), range, h, &opts)
.await
}
#[instrument(level = "debug", skip(self, data))]
pub(super) async fn handle_put_object(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
data: &mut ChunkNativePutData,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
check_put_object_args(bucket, object)?;
+1
View File
@@ -31,6 +31,7 @@ use rustfs_filemeta::{
RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
version_purge_statuses_map,
};
use rustfs_io_core::BoxChunkStream;
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_rio::Checksum;
+202 -25
View File
@@ -1,7 +1,101 @@
use super::*;
use rustfs_rio::TryGetIndex;
pub struct ChunkNativePutData {
stream: Option<HashReader>,
size: i64,
actual_size: i64,
}
impl Debug for ChunkNativePutData {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ChunkNativePutData").finish()
}
}
impl ChunkNativePutData {
pub fn new(stream: HashReader) -> Self {
let size = stream.size();
let actual_size = stream.actual_size();
Self {
stream: Some(stream),
size,
actual_size,
}
}
pub fn from_vec(data: Vec<u8>) -> Self {
use sha2::{Digest, Sha256};
let content_length = data.len() as i64;
let sha256hex = if content_length > 0 {
Some(hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower))
} else {
None
};
Self::new(HashReader::from_stream(Cursor::new(data), content_length, content_length, None, sha256hex, false).unwrap())
}
pub fn take_stream(&mut self) -> std::io::Result<HashReader> {
self.stream
.take()
.ok_or_else(|| std::io::Error::other("ChunkNativePutData stream already taken"))
}
pub fn restore_stream(&mut self, stream: HashReader) {
self.size = stream.size();
self.actual_size = stream.actual_size();
self.stream = Some(stream);
}
pub fn as_hash_reader(&self) -> Option<&HashReader> {
self.stream.as_ref()
}
pub fn as_hash_reader_mut(&mut self) -> Option<&mut HashReader> {
self.stream.as_mut()
}
pub fn index_bytes(&self) -> Option<Bytes> {
self.as_hash_reader()
.and_then(|reader| reader.try_get_index().map(|index| index.clone().into_vec()))
}
pub fn resolve_etag(&mut self) -> Option<String> {
self.as_hash_reader_mut()
.and_then(rustfs_rio::EtagResolvable::try_resolve_etag)
}
pub fn content_hash_bytes(&mut self) -> std::io::Result<Option<Bytes>> {
let Some(reader) = self.as_hash_reader_mut() else {
return Ok(None);
};
Ok(reader
.finalize_content_hash()?
.as_ref()
.map(|checksum| checksum.to_bytes(&[])))
}
pub fn content_crc_type(&self) -> Option<rustfs_rio::ChecksumType> {
self.as_hash_reader().and_then(HashReader::content_crc_type)
}
pub fn content_crc(&self) -> HashMap<String, String> {
self.as_hash_reader().map_or_else(HashMap::new, HashReader::content_crc)
}
pub fn size(&self) -> i64 {
self.size
}
pub fn actual_size(&self) -> i64 {
self.actual_size
}
}
pub struct PutObjReader {
pub stream: HashReader,
data: ChunkNativePutData,
}
impl Debug for PutObjReader {
@@ -12,32 +106,81 @@ impl Debug for PutObjReader {
impl PutObjReader {
pub fn new(stream: HashReader) -> Self {
PutObjReader { stream }
Self {
data: ChunkNativePutData::new(stream),
}
}
pub fn as_hash_reader(&self) -> &HashReader {
&self.stream
pub fn chunk_native_data(&self) -> &ChunkNativePutData {
&self.data
}
pub fn chunk_native_data_mut(&mut self) -> &mut ChunkNativePutData {
&mut self.data
}
pub fn take_stream(&mut self) -> std::io::Result<HashReader> {
self.data.take_stream()
}
pub fn restore_stream(&mut self, stream: HashReader) {
self.data.restore_stream(stream);
}
pub fn as_hash_reader(&self) -> Option<&HashReader> {
self.data.as_hash_reader()
}
pub fn as_hash_reader_mut(&mut self) -> Option<&mut HashReader> {
self.data.as_hash_reader_mut()
}
pub fn index_bytes(&self) -> Option<Bytes> {
self.data.index_bytes()
}
pub fn resolve_etag(&mut self) -> Option<String> {
self.data.resolve_etag()
}
pub fn content_hash_bytes(&mut self) -> std::io::Result<Option<Bytes>> {
self.data.content_hash_bytes()
}
pub fn content_crc_type(&self) -> Option<rustfs_rio::ChecksumType> {
self.data.content_crc_type()
}
pub fn content_crc(&self) -> HashMap<String, String> {
self.data.content_crc()
}
pub fn from_vec(data: Vec<u8>) -> Self {
use sha2::{Digest, Sha256};
let content_length = data.len() as i64;
let sha256hex = if content_length > 0 {
Some(hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower))
} else {
None
};
PutObjReader {
stream: HashReader::from_stream(Cursor::new(data), content_length, content_length, None, sha256hex, false).unwrap(),
Self {
data: ChunkNativePutData::from_vec(data),
}
}
pub fn size(&self) -> i64 {
self.stream.size()
self.data.size()
}
pub fn actual_size(&self) -> i64 {
self.stream.actual_size()
self.data.actual_size()
}
}
impl std::ops::Deref for PutObjReader {
type Target = ChunkNativePutData;
fn deref(&self) -> &Self::Target {
&self.data
}
}
impl std::ops::DerefMut for PutObjReader {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.data
}
}
@@ -46,6 +189,26 @@ pub struct GetObjectReader {
pub object_info: ObjectInfo,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GetObjectChunkPath {
Direct,
Bridge,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GetObjectChunkCopyMode {
TrueZeroCopy,
SharedBytes,
SingleCopy,
Reconstructed,
}
pub struct GetObjectChunkResult {
pub stream: BoxChunkStream,
pub path: GetObjectChunkPath,
pub copy_mode: GetObjectChunkCopyMode,
}
impl GetObjectReader {
#[tracing::instrument(level = "debug", skip(reader, rs, opts, _h))]
pub fn new(
@@ -63,31 +226,34 @@ impl GetObjectReader {
rs = HTTPRangeSpec::from_object_info(oi, part_number);
}
// TODO:Encrypted
let logical_size = oi.get_actual_size()?;
let encrypted_object = oi.user_defined.contains_key("x-rustfs-encryption-key")
|| oi
.user_defined
.contains_key("x-amz-server-side-encryption-customer-algorithm");
let (algo, is_compressed) = oi.is_compressed_ok()?;
// TODO: check TRANSITION
if is_compressed {
let actual_size = oi.get_actual_size()?;
let (off, length, dec_off, dec_length) = if let Some(rs) = rs {
// Support range requests for compressed objects
let (dec_off, dec_length) = rs.get_offset_length(actual_size)?;
let (dec_off, dec_length) = rs.get_offset_length(logical_size)?;
(0, oi.size, dec_off, dec_length)
} else {
(0, oi.size, 0, actual_size)
(0, oi.size, 0, logical_size)
};
let dec_reader = DecompressReader::new(reader, algo);
let actual_size_usize = if actual_size > 0 {
actual_size as usize
let actual_size_usize = if logical_size > 0 {
logical_size as usize
} else {
return Err(Error::other(format!("invalid decompressed size {actual_size}")));
return Err(Error::other(format!("invalid decompressed size {logical_size}")));
};
let final_reader: Box<dyn AsyncRead + Unpin + Send + Sync> = if dec_off > 0 || dec_length != actual_size {
let final_reader: Box<dyn AsyncRead + Unpin + Send + Sync> = if dec_off > 0 || dec_length != logical_size {
// Use RangedDecompressReader for streaming range processing
// The new implementation supports any offset size by streaming and skipping data
match RangedDecompressReader::new(dec_reader, dec_off, dec_length, actual_size_usize) {
@@ -122,8 +288,19 @@ impl GetObjectReader {
));
}
if encrypted_object && rs.is_none() {
return Ok((
GetObjectReader {
stream: reader,
object_info: oi.clone(),
},
0,
oi.size,
));
}
if let Some(rs) = rs {
let (off, length) = rs.get_offset_length(oi.size)?;
let (off, length) = rs.get_offset_length(logical_size)?;
Ok((
GetObjectReader {
@@ -140,7 +317,7 @@ impl GetObjectReader {
object_info: oi.clone(),
},
0,
oi.size,
logical_size,
))
}
}
+8 -2
View File
@@ -11,7 +11,13 @@ pub trait ObjectIO: Send + Sync + Debug + 'static {
opts: &ObjectOptions,
) -> Result<GetObjectReader>;
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo>;
async fn put_object(
&self,
bucket: &str,
object: &str,
data: &mut ChunkNativePutData,
opts: &ObjectOptions,
) -> Result<ObjectInfo>;
}
/// Bucket-level storage operations.
@@ -126,7 +132,7 @@ pub trait MultipartOperations: Send + Sync + Debug {
object: &str,
upload_id: &str,
part_id: usize,
data: &mut PutObjReader,
data: &mut ChunkNativePutData,
opts: &ObjectOptions,
) -> Result<PartInfo>;
async fn get_multipart_info(
+15 -1
View File
@@ -70,6 +70,7 @@ pub struct ObjectOptions {
pub eval_metadata: Option<HashMap<String, String>>,
pub resolved_checksum: Option<Bytes>,
pub want_checksum: Option<Checksum>,
pub skip_verify_bitrot: bool,
}
@@ -283,7 +284,7 @@ pub struct ObjectInfo {
pub expires: Option<OffsetDateTime>,
pub num_versions: usize,
pub successor_mod_time: Option<OffsetDateTime>,
pub put_object_reader: Option<PutObjReader>,
pub put_object_reader: Option<ChunkNativePutData>,
pub etag: Option<String>,
pub inlined: bool,
pub metadata_only: bool,
@@ -509,6 +510,18 @@ impl ObjectInfo {
})
.collect();
let actual_size = fi
.parts
.iter()
.map(|part| {
if part.actual_size > 0 {
part.actual_size
} else {
i64::try_from(part.size).unwrap_or_default()
}
})
.sum();
// TODO: part checksums
ObjectInfo {
@@ -521,6 +534,7 @@ impl ObjectInfo {
delete_marker: fi.deleted,
mod_time: fi.mod_time,
size: fi.size,
actual_size,
parts,
is_latest: fi.is_latest,
user_tags,
+7 -6
View File
@@ -51,7 +51,7 @@ use crate::{
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
global::is_first_cluster_node_local,
store::ECStore,
store_api::{ObjectIO as _, ObjectOptions, PutObjReader},
store_api::{ChunkNativePutData, ObjectIO as _, ObjectOptions},
};
use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
@@ -1046,9 +1046,8 @@ impl TierConfigMgr {
opts: &ObjectOptions,
) -> std::result::Result<(), std::io::Error> {
debug!("save tier config:{}", file);
let _ = api
.put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts)
.await?;
let mut put_data = ChunkNativePutData::from_vec(data.to_vec());
let _ = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await?;
Ok(())
}
@@ -1104,11 +1103,12 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
Ok(data) => {
let cfg = TierConfigMgr::unmarshal(&data)?;
let normalized = encode_external_tiering_config_blob(&cfg)?;
let mut put_data = ChunkNativePutData::from_vec(normalized.to_vec());
let _ = api
.put_object(
RUSTFS_META_BUCKET,
&config_file,
&mut PutObjReader::from_vec(normalized.to_vec()),
&mut put_data,
&ObjectOptions {
max_parity: true,
..Default::default()
@@ -1158,10 +1158,11 @@ async fn read_tier_config_from_bucket<S: StorageAPI>(
}
async fn write_tier_config_to_rustfs<S: StorageAPI>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()> {
let mut put_data = ChunkNativePutData::from_vec(data.to_vec());
api.put_object(
RUSTFS_META_BUCKET,
path,
&mut PutObjReader::from_vec(data.to_vec()),
&mut put_data,
&ObjectOptions {
max_parity: true,
..Default::default()
+5 -1
View File
@@ -24,7 +24,7 @@ use rustfs_utils::http::headers::{
AMZ_STORAGE_CLASS,
};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS,
AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_CRC, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS,
SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, has_internal_suffix, insert_bytes,
is_internal_key,
};
@@ -230,6 +230,10 @@ impl FileMeta {
}
}
if let Some(checksum) = fi.checksum.as_ref() {
insert_bytes(&mut obj.meta_sys, SUFFIX_CRC, checksum.to_vec());
}
if let Some(mod_time) = fi.mod_time {
obj.mod_time = Some(mod_time);
}
+1 -1
View File
@@ -208,7 +208,7 @@ impl HealStorageAPI for ECStoreHealStorage {
async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()> {
debug!("Putting object data: {}/{} ({} bytes)", bucket, object, data.len());
let mut reader = rustfs_ecstore::store_api::PutObjReader::from_vec(data.to_vec());
let mut reader = rustfs_ecstore::store_api::ChunkNativePutData::from_vec(data.to_vec());
match (*self.ecstore)
.put_object(bucket, object, &mut reader, &Default::default())
.await
+2 -2
View File
@@ -18,7 +18,7 @@ use rustfs_ecstore::{
disk::endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
store::ECStore,
store_api::{BucketOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
store_api::{BucketOperations, ChunkNativePutData, ObjectIO, ObjectOperations, ObjectOptions},
};
use rustfs_heal::heal::{
manager::{HealConfig, HealManager},
@@ -162,7 +162,7 @@ async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
/// Test helper: Upload test object
async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) {
let mut reader = PutObjReader::from_vec(data.to_vec());
let mut reader = ChunkNativePutData::from_vec(data.to_vec());
let object_info = (**ecstore)
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
+2
View File
@@ -29,12 +29,14 @@ workspace = true
[dependencies]
bytes = { workspace = true }
futures-core = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync"] }
memmap2 = { workspace = true }
rustfs-io-metrics = { workspace = true }
[dev-dependencies]
futures-util = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
[lib]
+124
View File
@@ -0,0 +1,124 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Compatibility adapter that exposes chunk streams as `AsyncRead`.
use crate::chunk::BoxChunkStream;
use bytes::Bytes;
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
/// `AsyncRead` adapter for boxed chunk streams.
pub struct ChunkStreamReader {
stream: BoxChunkStream,
current: Option<Bytes>,
offset: usize,
}
impl ChunkStreamReader {
#[must_use]
pub fn new(stream: BoxChunkStream) -> Self {
Self {
stream,
current: None,
offset: 0,
}
}
}
impl AsyncRead for ChunkStreamReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
loop {
if let Some(current) = &self.current {
if self.offset < current.len() {
let remaining = &current[self.offset..];
let to_read = remaining.len().min(buf.remaining());
buf.put_slice(&remaining[..to_read]);
self.offset += to_read;
return Poll::Ready(Ok(()));
}
self.current = None;
self.offset = 0;
continue;
}
match self.stream.as_mut().poll_next(cx) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Some(Ok(chunk))) => {
let next = chunk.as_bytes();
if next.is_empty() {
continue;
}
self.current = Some(next);
self.offset = 0;
}
Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)),
Poll::Ready(None) => return Poll::Ready(Ok(())),
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::chunk::{IoChunk, MappedChunk, PooledChunk};
use bytes::Bytes;
use futures_util::stream;
use tokio::io::AsyncReadExt;
#[tokio::test]
async fn test_chunk_stream_reader_reads_single_chunk() {
let stream: BoxChunkStream = Box::pin(stream::iter(vec![Ok(IoChunk::Shared(Bytes::from_static(b"hello")))]));
let mut reader = ChunkStreamReader::new(stream);
let mut out = Vec::new();
reader.read_to_end(&mut out).await.unwrap();
assert_eq!(out, b"hello");
}
#[tokio::test]
async fn test_chunk_stream_reader_reads_multiple_chunks() {
let stream: BoxChunkStream = Box::pin(stream::iter(vec![
Ok(IoChunk::Shared(Bytes::from_static(b"he"))),
Ok(IoChunk::Mapped(MappedChunk::new(Bytes::from_static(b"llo!"), 0, 4).unwrap())),
Ok(IoChunk::Pooled(PooledChunk::from_bytes(Bytes::from_static(b" world")).unwrap())),
]));
let mut reader = ChunkStreamReader::new(stream);
let mut out = Vec::new();
reader.read_to_end(&mut out).await.unwrap();
assert_eq!(out, b"hello! world");
}
#[tokio::test]
async fn test_chunk_stream_reader_handles_empty_stream() {
let stream: BoxChunkStream = Box::pin(stream::iter(Vec::<io::Result<IoChunk>>::new()));
let mut reader = ChunkStreamReader::new(stream);
let mut out = Vec::new();
reader.read_to_end(&mut out).await.unwrap();
assert!(out.is_empty());
}
#[tokio::test]
async fn test_chunk_stream_reader_propagates_stream_error() {
let stream: BoxChunkStream = Box::pin(stream::iter(vec![Err(io::Error::other("chunk stream failure"))]));
let mut reader = ChunkStreamReader::new(stream);
let mut out = Vec::new();
let err = reader.read_to_end(&mut out).await.unwrap_err();
assert_eq!(err.kind(), io::ErrorKind::Other);
assert!(err.to_string().contains("chunk stream failure"));
}
}
+276
View File
@@ -0,0 +1,276 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Core chunk ownership abstractions for the zero-copy data plane.
use crate::pool::PooledBuffer;
use bytes::Bytes;
use futures_core::Stream;
use std::io;
use std::pin::Pin;
/// Boxed asynchronous stream of I/O chunks.
pub type BoxChunkStream = Pin<Box<dyn Stream<Item = io::Result<IoChunk>> + Send + Sync + 'static>>;
/// Source of chunked data.
pub trait ChunkSource {
fn into_chunk_stream(self) -> BoxChunkStream
where
Self: Sized;
}
/// Owned chunk variants used by the zero-copy data plane.
#[derive(Debug)]
pub enum IoChunk {
Shared(Bytes),
Mapped(MappedChunk),
Pooled(PooledChunk),
}
impl IoChunk {
/// Returns the visible length of this chunk.
#[must_use]
pub fn len(&self) -> usize {
match self {
Self::Shared(bytes) => bytes.len(),
Self::Mapped(chunk) => chunk.len(),
Self::Pooled(chunk) => chunk.len(),
}
}
/// Returns true when the chunk has no visible data.
#[must_use]
pub fn is_empty(&self) -> bool {
self.len() == 0
}
/// Returns a shared read-only view of the visible bytes.
#[must_use]
pub fn as_bytes(&self) -> Bytes {
match self {
Self::Shared(bytes) => bytes.clone(),
Self::Mapped(chunk) => chunk.as_bytes(),
Self::Pooled(chunk) => chunk.as_bytes(),
}
}
/// Returns a sliced view relative to the currently visible bytes.
pub fn slice(&self, offset: usize, len: usize) -> io::Result<Self> {
match self {
Self::Shared(bytes) => {
validate_slice_bounds(bytes.len(), offset, len)?;
Ok(Self::Shared(bytes.slice(offset..offset + len)))
}
Self::Mapped(chunk) => chunk.slice(offset, len).map(Self::Mapped),
Self::Pooled(chunk) => chunk.slice(offset, len).map(Self::Pooled),
}
}
}
/// Logical view into mapped file bytes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MappedChunk {
bytes: Bytes,
logical_offset: usize,
logical_len: usize,
}
impl MappedChunk {
pub fn new(bytes: Bytes, logical_offset: usize, logical_len: usize) -> io::Result<Self> {
validate_slice_bounds(bytes.len(), logical_offset, logical_len)?;
Ok(Self {
bytes,
logical_offset,
logical_len,
})
}
/// Returns the visible length of this mapped chunk.
#[must_use]
pub const fn len(&self) -> usize {
self.logical_len
}
/// Returns true when the chunk has no visible data.
#[must_use]
pub const fn is_empty(&self) -> bool {
self.logical_len == 0
}
/// Returns the visible bytes for this logical view.
#[must_use]
pub fn as_bytes(&self) -> Bytes {
self.bytes
.slice(self.logical_offset..self.logical_offset.saturating_add(self.logical_len))
}
/// Returns a sliced logical view relative to the current logical view.
pub fn slice(&self, offset: usize, len: usize) -> io::Result<Self> {
validate_slice_bounds(self.logical_len, offset, len)?;
Self::new(self.bytes.clone(), self.logical_offset + offset, len)
}
}
/// Placeholder pooled chunk variant for commit 4.
///
/// This is backed by a `PooledBuffer` and exposes a visible read-only window.
#[derive(Debug)]
pub struct PooledChunk {
bytes: Bytes,
}
#[derive(Debug)]
struct PooledChunkOwner {
buffer: PooledBuffer,
visible_len: usize,
}
impl AsRef<[u8]> for PooledChunkOwner {
fn as_ref(&self) -> &[u8] {
&self.buffer[..self.visible_len]
}
}
#[derive(Debug)]
struct DetachedVecChunkOwner {
bytes: Vec<u8>,
}
impl AsRef<[u8]> for DetachedVecChunkOwner {
fn as_ref(&self) -> &[u8] {
&self.bytes
}
}
impl PooledChunk {
pub fn new(buffer: PooledBuffer, len: usize) -> io::Result<Self> {
validate_slice_bounds(buffer.len(), 0, len)?;
Ok(Self {
bytes: Bytes::from_owner(PooledChunkOwner {
buffer,
visible_len: len,
}),
})
}
/// Convenience constructor for detached test and compatibility values.
pub fn from_bytes(bytes: Bytes) -> io::Result<Self> {
let len = bytes.len();
Self::new(PooledBuffer::from_bytes(bytes), len)
}
/// Detached constructor that takes ownership of an existing `Vec<u8>`
/// without introducing an additional copy.
pub fn from_vec(bytes: Vec<u8>) -> Self {
Self {
bytes: Bytes::from_owner(DetachedVecChunkOwner { bytes }),
}
}
/// Returns the visible length of this pooled chunk.
#[must_use]
pub fn len(&self) -> usize {
self.bytes.len()
}
/// Returns true when the chunk has no visible data.
#[must_use]
pub fn is_empty(&self) -> bool {
self.bytes.is_empty()
}
/// Returns the visible bytes for this pooled chunk.
#[must_use]
pub fn as_bytes(&self) -> Bytes {
self.bytes.clone()
}
/// Returns a sliced pooled chunk relative to the current visible view.
pub fn slice(&self, offset: usize, len: usize) -> io::Result<Self> {
validate_slice_bounds(self.bytes.len(), offset, len)?;
Ok(Self {
bytes: self.bytes.slice(offset..offset + len),
})
}
}
fn validate_slice_bounds(visible_len: usize, offset: usize, len: usize) -> io::Result<()> {
let end = offset
.checked_add(len)
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "chunk slice overflows"))?;
if end > visible_len {
return Err(io::Error::new(io::ErrorKind::InvalidInput, "chunk slice exceeds visible length"));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::pool::BytesPool;
#[test]
fn test_shared_chunk_len_and_slice() {
let chunk = IoChunk::Shared(Bytes::from_static(b"abcdef"));
assert_eq!(chunk.len(), 6);
assert!(!chunk.is_empty());
assert_eq!(chunk.as_bytes(), Bytes::from_static(b"abcdef"));
assert_eq!(chunk.slice(1, 3).unwrap().as_bytes(), Bytes::from_static(b"bcd"));
}
#[test]
fn test_mapped_chunk_len_and_slice() {
let chunk = MappedChunk::new(Bytes::from_static(b"abcdefgh"), 2, 4).unwrap();
assert_eq!(chunk.len(), 4);
assert_eq!(chunk.as_bytes(), Bytes::from_static(b"cdef"));
assert_eq!(chunk.slice(1, 2).unwrap().as_bytes(), Bytes::from_static(b"de"));
}
#[test]
fn test_pooled_chunk_len_and_as_bytes() {
let chunk = PooledChunk::from_bytes(Bytes::from_static(b"hello")).unwrap();
assert_eq!(chunk.len(), 5);
assert_eq!(chunk.as_bytes(), Bytes::from_static(b"hello"));
assert_eq!(chunk.slice(1, 3).unwrap().as_bytes(), Bytes::from_static(b"ell"));
}
#[test]
fn test_io_chunk_as_bytes_for_all_variants() {
let shared = IoChunk::Shared(Bytes::from_static(b"s"));
let mapped = IoChunk::Mapped(MappedChunk::new(Bytes::from_static(b"mapped"), 0, 6).unwrap());
let pooled = IoChunk::Pooled(PooledChunk::from_bytes(Bytes::from_static(b"p")).unwrap());
assert_eq!(shared.as_bytes(), Bytes::from_static(b"s"));
assert_eq!(mapped.as_bytes(), Bytes::from_static(b"mapped"));
assert_eq!(pooled.as_bytes(), Bytes::from_static(b"p"));
}
#[tokio::test]
async fn test_pooled_chunk_keeps_owner_alive_until_last_view_drops() {
let pool = BytesPool::new_tiered();
let mut buffer = pool.acquire_buffer(16).await;
buffer.extend_from_slice(b"pooled-bytes");
let chunk = PooledChunk::new(buffer, "pooled-bytes".len()).unwrap();
let bytes = chunk.as_bytes();
assert_eq!(pool.available_buffers(), 0);
drop(chunk);
assert_eq!(pool.available_buffers(), 0);
assert_eq!(bytes, Bytes::from_static(b"pooled-bytes"));
drop(bytes);
assert_eq!(pool.available_buffers(), 1);
}
}
+4
View File
@@ -46,8 +46,10 @@
//! let mut buffer = pool.acquire_buffer(8192).await;
//! ```
pub mod adapter;
pub mod backpressure;
pub mod bufreader_optimizer;
pub mod chunk;
pub mod config;
pub mod deadlock_detector;
pub mod direct_io;
@@ -68,7 +70,9 @@ pub use reader::{ZeroCopyObjectReader, ZeroCopyReadError};
pub use writer::{ZeroCopyObjectWriter, ZeroCopyWriteError};
// BufReader optimizer exports
pub use adapter::ChunkStreamReader;
pub use bufreader_optimizer::{BufReaderConfig, BufReaderOptimizer, BufReaderStats, BufferedSource};
pub use chunk::{BoxChunkStream, ChunkSource, IoChunk, MappedChunk, PooledChunk};
// Shared memory exports
pub use shared_memory::{ArcData, ArcMetadata, SharedMemoryConfig, SharedMemoryPool, SharedMemoryStats};
+41 -1
View File
@@ -17,7 +17,7 @@
//! Migrated from rustfs-ecstore to provide unified buffer pooling
//! across rustfs and rustfs-ecstore without cyclic dependencies.
use bytes::BytesMut;
use bytes::{Bytes, BytesMut};
use std::mem::ManuallyDrop;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
@@ -108,6 +108,7 @@ pub struct BytesPoolMetrics {
/// A buffer managed by the BytesPool.
///
/// When dropped, the buffer is automatically returned to the pool for reuse.
#[derive(Debug)]
pub struct PooledBuffer {
/// The underlying buffer (ManuallyDrop to allow taking on drop)
pub buffer: ManuallyDrop<BytesMut>,
@@ -117,6 +118,45 @@ pub struct PooledBuffer {
_permit: Option<OwnedSemaphorePermit>,
}
impl PooledBuffer {
/// Create a detached pooled buffer from bytes.
///
/// This is primarily used for tests and transitional adapters where the
/// chunk abstraction needs a pool-shaped owner before a real pool-backed
/// producer exists.
#[must_use]
pub fn from_bytes(bytes: Bytes) -> Self {
Self {
buffer: ManuallyDrop::new(BytesMut::from(bytes.as_ref())),
tier: None,
_permit: None,
}
}
/// Current visible length of the underlying buffer.
#[must_use]
pub fn len(&self) -> usize {
self.buffer.len()
}
/// Total buffer capacity.
#[must_use]
pub fn capacity(&self) -> usize {
self.buffer.capacity()
}
/// Clear the visible contents while preserving capacity.
pub fn clear(&mut self) {
self.buffer.clear();
}
/// Returns true when the visible buffer is empty.
#[must_use]
pub fn is_empty(&self) -> bool {
self.buffer.is_empty()
}
}
/// BytesPool configuration.
///
/// Allows customization of buffer sizes and limits for each tier.
+373 -225
View File
@@ -118,8 +118,129 @@ pub use config::{
// Re-exports for convenience
pub use collector::MetricsCollector;
pub use metric_names::data_plane;
pub use performance::PerformanceMetrics;
/// High-level request path selected for an I/O operation.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IoPath {
Fast,
Legacy,
}
impl IoPath {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Fast => "fast",
Self::Legacy => "legacy",
}
}
}
/// Effective copy mode observed for an I/O operation.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CopyMode {
TrueZeroCopy,
SharedBytes,
SingleCopy,
Reconstructed,
Transformed,
}
impl CopyMode {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::TrueZeroCopy => "true_zero_copy",
Self::SharedBytes => "shared_bytes",
Self::SingleCopy => "single_copy",
Self::Reconstructed => "reconstructed",
Self::Transformed => "transformed",
}
}
}
/// Stage where a data plane decision or fallback happened.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IoStage {
Unknown,
ReadSetup,
HttpBridge,
CacheWriteback,
LocalDiskChunk,
RangeGuard,
PutTransform,
}
impl IoStage {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Unknown => "unknown",
Self::ReadSetup => "read_setup",
Self::HttpBridge => "http_bridge",
Self::CacheWriteback => "cache_writeback",
Self::LocalDiskChunk => "local_disk_chunk",
Self::RangeGuard => "range_guard",
Self::PutTransform => "put_transform",
}
}
}
/// Reason why the data plane fell back from a preferred path.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FallbackReason {
Unknown,
MmapDisabled,
MmapUnavailable,
SmallObject,
WindowLimitExceeded,
UnalignedWindow,
RangeNotSupported,
EncryptionEnabled,
CompressionEnabled,
TransformEncryptionLegacy,
TransformCompressionLegacy,
TransformCompressionEncryptionLegacy,
ChunkBridgeUnavailable,
NonLocalBackend,
}
impl FallbackReason {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Unknown => "unknown",
Self::MmapDisabled => "mmap_disabled",
Self::MmapUnavailable => "mmap_unavailable",
Self::SmallObject => "small_object",
Self::WindowLimitExceeded => "window_limit_exceeded",
Self::UnalignedWindow => "unaligned_window",
Self::RangeNotSupported => "range_not_supported",
Self::EncryptionEnabled => "encryption_enabled",
Self::CompressionEnabled => "compression_enabled",
Self::TransformEncryptionLegacy => "transform_encryption_legacy",
Self::TransformCompressionLegacy => "transform_compression_legacy",
Self::TransformCompressionEncryptionLegacy => "transform_compression_encryption_legacy",
Self::ChunkBridgeUnavailable => "chunk_bridge_unavailable",
Self::NonLocalBackend => "non_local_backend",
}
}
}
#[inline(always)]
fn put_size_bucket_label(size_bytes: i64) -> &'static str {
match size_bytes {
..=0 => "unknown",
1..=16_384 => "le_16kib",
16_385..=65_536 => "le_64kib",
65_537..=262_144 => "le_256kib",
262_145..=1_048_576 => "le_1mib",
_ => "gt_1mib",
}
}
/// Record GetObject request start.
#[inline(always)]
pub fn record_get_object_request_start(concurrent_requests: usize) {
@@ -206,40 +327,138 @@ pub fn record_object_cache_writeback() {
counter!("rustfs_io_object_cache_writeback_total").increment(1);
}
/// Record a zero-copy read operation.
///
/// # Arguments
///
/// * `size_bytes` - Size of the data read in bytes
/// * `duration_ms` - Time taken for the read operation in milliseconds
/// Record which request path was selected for an operation.
#[inline(always)]
pub fn record_zero_copy_read(size_bytes: usize, duration_ms: f64) {
counter!("rustfs.zero_copy.reads.total").increment(1);
histogram!("rustfs.zero_copy.read.size.bytes").record(size_bytes as f64);
histogram!("rustfs.zero_copy.read.duration.ms").record(duration_ms);
pub fn record_io_path_selected(operation: &'static str, io_path: IoPath) {
counter!(
metric_names::data_plane::PATH_SELECTED_TOTAL,
"path" => operation,
"mode" => io_path.as_str()
)
.increment(1);
}
/// Record memory copies avoided by using zero-copy.
///
/// # Arguments
///
/// * `bytes_saved` - Number of bytes that would have been copied without zero-copy
/// Record the effective copy mode for an operation.
#[inline(always)]
pub fn record_memory_copy_saved(bytes_saved: usize) {
counter!("rustfs.zero_copy.memory.saved.bytes").increment(bytes_saved as u64);
pub fn record_io_copy_mode(operation: &'static str, copy_mode: CopyMode, size_bytes: usize) {
counter!(
metric_names::data_plane::COPY_MODE_BYTES_TOTAL,
"path" => operation,
"mode" => copy_mode.as_str()
)
.increment(size_bytes as u64);
}
/// Record a fallback from zero-copy to regular read.
///
/// This happens when zero-copy read fails (e.g., mmap not available,
/// file too large, etc.) and the system falls back to regular I/O.
///
/// # Arguments
///
/// * `reason` - Reason for the fallback (e.g., "mmap_unavailable", "file_too_large")
/// Record a data plane fallback decision.
#[inline(always)]
pub fn record_zero_copy_fallback(reason: &str) {
counter!("rustfs.zero_copy.fallback.total", "reason" => reason.to_string()).increment(1);
pub fn record_io_fallback(stage: IoStage, reason: FallbackReason) {
counter!(
metric_names::data_plane::FALLBACK_TOTAL,
"stage" => stage.as_str(),
"reason" => reason.as_str()
)
.increment(1);
}
/// Record the currently active mmap bytes held by LocalDisk chunk streams.
#[inline(always)]
pub fn record_local_disk_active_mmap_bytes(active_bytes: usize) {
gauge!(metric_names::data_plane::LOCAL_DISK_ACTIVE_MMAP_BYTES).set(active_bytes as f64);
}
/// Record pooled chunk usage in LocalDisk compatibility paths.
#[inline(always)]
pub fn record_local_disk_pooled_chunk(source: &'static str, size_bytes: usize) {
counter!(
metric_names::data_plane::LOCAL_DISK_POOLED_CHUNKS_TOTAL,
"source" => source
)
.increment(1);
counter!(
metric_names::data_plane::LOCAL_DISK_POOLED_BYTES_TOTAL,
"source" => source
)
.increment(size_bytes as u64);
}
/// Record a compatibility chunk-stream aggregation performed by `read_file_zero_copy()`.
#[inline(always)]
pub fn record_local_disk_compat_collect(chunk_count: usize, total_bytes: usize) {
counter!(metric_names::data_plane::LOCAL_DISK_COMPAT_COLLECT_TOTAL).increment(1);
histogram!(metric_names::data_plane::LOCAL_DISK_COMPAT_COLLECT_CHUNKS).record(chunk_count as f64);
histogram!(metric_names::data_plane::LOCAL_DISK_COMPAT_COLLECT_BYTES).record(total_bytes as f64);
}
/// Record an attempted PUT fast path.
#[inline(always)]
pub fn record_put_object_attempted_fast_path(size_bytes: i64) {
counter!(metric_names::data_plane::PUT_FAST_PATH_ATTEMPTS_TOTAL).increment(1);
if size_bytes > 0 {
histogram!(metric_names::data_plane::PUT_FAST_PATH_ATTEMPT_SIZE_BYTES).record(size_bytes as f64);
}
}
/// Record which transformed PUT pipeline was selected.
#[inline(always)]
pub fn record_put_transform_selected(kind: &'static str, io_path: IoPath, size_bytes: usize) {
counter!(
metric_names::data_plane::PUT_TRANSFORM_SELECTED_TOTAL,
"kind" => kind,
"mode" => io_path.as_str()
)
.increment(1);
histogram!(
metric_names::data_plane::PUT_TRANSFORM_SIZE_BYTES,
"kind" => kind,
"mode" => io_path.as_str()
)
.record(size_bytes as f64);
}
/// Record PUT path selection with size-bucket context.
#[inline(always)]
pub fn record_put_path_selected(size_bytes: i64, io_path: IoPath) {
counter!(
"rustfs.s3.put_object.path.selected.total",
"mode" => io_path.as_str(),
"size_bucket" => put_size_bucket_label(size_bytes)
)
.increment(1);
}
/// Record PUT copy mode with size-bucket context.
#[inline(always)]
pub fn record_put_copy_mode(size_bytes: i64, copy_mode: CopyMode) {
counter!(
"rustfs.s3.put_object.copy_mode.total",
"mode" => copy_mode.as_str(),
"size_bucket" => put_size_bucket_label(size_bytes)
)
.increment(1);
}
/// Record PUT fallback with size-bucket context.
#[inline(always)]
pub fn record_put_fallback(size_bytes: i64, reason: FallbackReason) {
counter!(
"rustfs.s3.put_object.fallback.total",
"reason" => reason.as_str(),
"size_bucket" => put_size_bucket_label(size_bytes)
)
.increment(1);
}
/// Record inline-object selection for PUT with size-bucket context.
#[inline(always)]
pub fn record_put_inline_selected(size_bytes: i64, versioned: bool) {
counter!(
"rustfs.s3.put_object.inline.selected.total",
"versioned" => if versioned { "true" } else { "false" },
"size_bucket" => put_size_bucket_label(size_bytes)
)
.increment(1);
}
// ============================================================================
@@ -297,41 +516,6 @@ pub fn record_bytes_pool_hit_rate(tier: &str, hit_rate: f64) {
gauge!("rustfs.bytes.pool.hit.rate", "tier" => tier.to_string()).set(hit_rate * 100.0);
}
/// Record zero-copy write operation.
///
/// # Arguments
///
/// * `size_bytes` - Size of the data written in bytes
/// * `duration_ms` - Time taken for the write operation in milliseconds
#[inline(always)]
pub fn record_zero_copy_write(size_bytes: usize, duration_ms: f64) {
counter!("rustfs.zero_copy.write.total").increment(1);
histogram!("rustfs.zero_copy.write.size.bytes").record(size_bytes as f64);
histogram!("rustfs.zero_copy.write.duration.ms").record(duration_ms);
}
/// Record zero-copy write fallback.
///
/// This happens when zero-copy write fails and the system falls back to regular I/O.
///
/// # Arguments
///
/// * `reason` - Reason for the fallback
#[inline(always)]
pub fn record_zero_copy_write_fallback(reason: &str) {
counter!("rustfs.zero_copy.write.fallback.total", "reason" => reason.to_string()).increment(1);
}
/// Record bytes saved from zero-copy.
///
/// # Arguments
///
/// * `size_bytes` - Number of bytes saved from zero-copy
#[inline(always)]
pub fn record_bytes_saved(size_bytes: usize) {
counter!("rustfs.zero_copy.bytes.saved.total").increment(size_bytes as u64);
}
// ============================================================================
// S3 Operation Metrics (GetObject, PutObject, etc.)
// ============================================================================
@@ -343,6 +527,9 @@ pub fn record_bytes_saved(size_bytes: usize) {
/// * `duration_ms` - Operation duration in milliseconds
/// * `size_bytes` - Object size in bytes
/// * `from_cache` - Whether the object was served from cache
///
/// Note: this function records aggregate S3 GET metrics only. It must not be
/// interpreted as the definitive source of truth for data-plane copy mode.
#[inline(always)]
pub fn record_get_object(duration_ms: f64, size_bytes: i64, from_cache: bool) {
counter!("rustfs.s3.get_object.total").increment(1);
@@ -365,14 +552,33 @@ pub fn record_get_object(duration_ms: f64, size_bytes: i64, from_cache: bool) {
///
/// * `duration_ms` - Operation duration in milliseconds
/// * `size_bytes` - Object size in bytes
/// * `zero_copy_enabled` - Whether zero-copy was enabled for this operation
/// * `zero_copy_enabled` - Legacy aggregate flag preserved for compatibility
///
/// Note: this function records aggregate S3 PUT metrics only. The definitive
/// outcome of request-level fast-path attempts must be tracked separately via
/// ADR 0001 data-plane helpers.
#[inline(always)]
pub fn record_put_object(duration_ms: f64, size_bytes: i64, zero_copy_enabled: bool) {
counter!("rustfs.s3.put_object.total").increment(1);
histogram!("rustfs.s3.put_object.duration.ms").record(duration_ms);
counter!(
"rustfs.s3.put_object.bucketed.total",
"size_bucket" => put_size_bucket_label(size_bytes)
)
.increment(1);
histogram!(
"rustfs.s3.put_object.bucketed.duration.ms",
"size_bucket" => put_size_bucket_label(size_bytes)
)
.record(duration_ms);
if size_bytes > 0 {
histogram!("rustfs.s3.put_object.size.bytes").record(size_bytes as f64);
histogram!(
"rustfs.s3.put_object.bucketed.size.bytes",
"size_bucket" => put_size_bucket_label(size_bytes)
)
.record(size_bytes as f64);
}
if zero_copy_enabled {
@@ -717,10 +923,110 @@ mod tests {
use super::*;
#[test]
fn test_record_zero_copy_read() {
record_zero_copy_read(1024, 10.5);
record_memory_copy_saved(1024);
record_zero_copy_fallback("test");
fn test_io_path_as_str_values_stable() {
assert_eq!(IoPath::Fast.as_str(), "fast");
assert_eq!(IoPath::Legacy.as_str(), "legacy");
}
#[test]
fn test_copy_mode_as_str_values_stable() {
assert_eq!(CopyMode::TrueZeroCopy.as_str(), "true_zero_copy");
assert_eq!(CopyMode::SharedBytes.as_str(), "shared_bytes");
assert_eq!(CopyMode::SingleCopy.as_str(), "single_copy");
assert_eq!(CopyMode::Reconstructed.as_str(), "reconstructed");
assert_eq!(CopyMode::Transformed.as_str(), "transformed");
}
#[test]
fn test_fallback_reason_as_str_values_stable() {
assert_eq!(FallbackReason::Unknown.as_str(), "unknown");
assert_eq!(FallbackReason::MmapDisabled.as_str(), "mmap_disabled");
assert_eq!(FallbackReason::MmapUnavailable.as_str(), "mmap_unavailable");
assert_eq!(FallbackReason::SmallObject.as_str(), "small_object");
assert_eq!(FallbackReason::WindowLimitExceeded.as_str(), "window_limit_exceeded");
assert_eq!(FallbackReason::UnalignedWindow.as_str(), "unaligned_window");
assert_eq!(FallbackReason::RangeNotSupported.as_str(), "range_not_supported");
assert_eq!(FallbackReason::EncryptionEnabled.as_str(), "encryption_enabled");
assert_eq!(FallbackReason::CompressionEnabled.as_str(), "compression_enabled");
assert_eq!(FallbackReason::TransformEncryptionLegacy.as_str(), "transform_encryption_legacy");
assert_eq!(FallbackReason::TransformCompressionLegacy.as_str(), "transform_compression_legacy");
assert_eq!(
FallbackReason::TransformCompressionEncryptionLegacy.as_str(),
"transform_compression_encryption_legacy"
);
assert_eq!(FallbackReason::ChunkBridgeUnavailable.as_str(), "chunk_bridge_unavailable");
assert_eq!(FallbackReason::NonLocalBackend.as_str(), "non_local_backend");
}
#[test]
fn test_record_io_path_selected() {
record_io_path_selected("get", IoPath::Fast);
record_io_path_selected("put", IoPath::Legacy);
}
#[test]
fn test_record_io_copy_mode() {
record_io_copy_mode("get", CopyMode::SharedBytes, 1024);
record_io_copy_mode("put", CopyMode::Transformed, 2048);
}
#[test]
fn test_record_io_fallback() {
record_io_fallback(IoStage::ReadSetup, FallbackReason::MmapUnavailable);
record_io_fallback(IoStage::HttpBridge, FallbackReason::ChunkBridgeUnavailable);
}
#[test]
fn test_record_local_disk_active_mmap_bytes() {
record_local_disk_active_mmap_bytes(4096);
record_local_disk_active_mmap_bytes(0);
}
#[test]
fn test_record_local_disk_pooled_chunk() {
record_local_disk_pooled_chunk("fallback", 4096);
record_local_disk_pooled_chunk("compat_collect", 8192);
}
#[test]
fn test_record_local_disk_compat_collect() {
record_local_disk_compat_collect(3, 16384);
}
#[test]
fn test_record_put_object_attempted_fast_path() {
record_put_object_attempted_fast_path(1024 * 1024);
record_put_object_attempted_fast_path(0);
}
#[test]
fn test_record_put_transform_selected() {
record_put_transform_selected("compression", IoPath::Fast, 2048);
record_put_transform_selected("compression_encryption", IoPath::Legacy, 4096);
}
#[test]
fn test_record_put_path_selected() {
record_put_path_selected(8 * 1024, IoPath::Fast);
record_put_path_selected(2 * 1024 * 1024, IoPath::Legacy);
}
#[test]
fn test_record_put_copy_mode() {
record_put_copy_mode(8 * 1024, CopyMode::SingleCopy);
record_put_copy_mode(512 * 1024, CopyMode::Transformed);
}
#[test]
fn test_record_put_fallback() {
record_put_fallback(32 * 1024, FallbackReason::CompressionEnabled);
record_put_fallback(2 * 1024 * 1024, FallbackReason::EncryptionEnabled);
}
#[test]
fn test_record_put_inline_selected() {
record_put_inline_selected(8 * 1024, false);
record_put_inline_selected(32 * 1024, true);
}
#[test]
@@ -731,13 +1037,6 @@ mod tests {
record_bytes_pool_hit_rate("small", 0.85);
}
#[test]
fn test_record_zero_copy_write() {
record_zero_copy_write(1024, 10.5);
record_zero_copy_write_fallback("test");
record_bytes_saved(1024);
}
// S3 Operation Metrics Tests
#[test]
fn test_record_get_object() {
@@ -857,157 +1156,6 @@ mod tests {
}
}
// ============================================================================
// Zero-Copy Optimization Metrics (Phase 1 Extension)
// ============================================================================
pub mod bandwidth;
pub mod global_metrics;
pub mod metric_names;
pub use metric_names::zero_copy;
/// Record a zero-copy buffer operation.
///
/// This function records metrics for zero-copy buffer operations,
/// including the operation type and size.
#[inline(always)]
pub fn record_zero_copy_buffer_operation(operation: &str, size: usize) {
counter!(
zero_copy::BUFFER_OPERATIONS_TOTAL,
"operation" => operation.to_string()
)
.increment(1);
counter!(
zero_copy::BUFFER_BYTES_TOTAL,
"operation" => operation.to_string()
)
.increment(size as u64);
}
/// Record memory copy operations.
///
/// This function tracks the number and size of memory copies,
/// which should be minimized in zero-copy paths.
#[inline(always)]
pub fn record_memory_copy(count: u32, size: usize) {
counter!(zero_copy::MEMORY_COPY_TOTAL).increment(count as u64);
counter!(zero_copy::MEMORY_COPY_BYTES_TOTAL).increment(size as u64);
histogram!("rustfs_memory_copy_size_bytes").record(size as f64);
}
/// Record a shared reference operation.
///
/// This function tracks operations that create or use shared references
/// for zero-copy data sharing.
#[inline(always)]
pub fn record_shared_ref_operation(operation: &str) {
counter!(
zero_copy::SHARED_REF_OPERATIONS_TOTAL,
"operation" => operation.to_string()
)
.increment(1);
}
/// Record BufReader optimization.
///
/// This function tracks BufReader layer elimination and buffer size
/// adjustments.
#[inline(always)]
pub fn record_bufreader_optimization(layers_eliminated: u32, buffer_size: usize) {
counter!(zero_copy::BUFREADER_LAYERS_ELIMINATED_TOTAL).increment(layers_eliminated as u64);
histogram!(zero_copy::BUFREADER_BUFFER_SIZE_BYTES).record(buffer_size as f64);
}
/// Record Direct I/O operation.
///
/// This function tracks Direct I/O operations and their success/fallback
/// status.
#[inline(always)]
pub fn record_direct_io_operation(operation: &str, size: usize, success: bool) {
let status = if success { "success" } else { "fallback" };
counter!(
zero_copy::DIRECT_IO_OPERATIONS_TOTAL,
"operation" => operation.to_string(),
"status" => status.to_string()
)
.increment(1);
counter!(
zero_copy::DIRECT_IO_BYTES_TOTAL,
"operation" => operation.to_string(),
"status" => status.to_string()
)
.increment(size as u64);
}
/// Update zero-copy performance metrics.
///
/// This function updates gauge metrics for overall zero-copy performance.
#[inline(always)]
pub fn update_zero_copy_performance_metrics(copy_count: u32, throughput_mbps: f64, memory_saved: u64) {
gauge!(zero_copy::AVG_COPY_COUNT).set(copy_count as f64);
gauge!(zero_copy::THROUGHPUT_MBPS).set(throughput_mbps);
gauge!(zero_copy::MEMORY_SAVED_BYTES).set(memory_saved as f64);
}
// ============================================================================
// Zero-Copy Metrics Tests
// ============================================================================
#[cfg(test)]
mod zero_copy_tests {
use super::*;
#[test]
fn test_record_zero_copy_buffer_operation() {
// This test verifies the function compiles and runs
// Actual metric verification requires a metrics recorder
record_zero_copy_buffer_operation("read", 1024);
record_zero_copy_buffer_operation("write", 2048);
}
#[test]
fn test_record_memory_copy() {
record_memory_copy(1, 1024);
record_memory_copy(2, 2048);
}
#[test]
fn test_record_shared_ref_operation() {
record_shared_ref_operation("create");
record_shared_ref_operation("share");
}
#[test]
fn test_record_bufreader_optimization() {
record_bufreader_optimization(1, 8192);
record_bufreader_optimization(2, 65536);
}
#[test]
fn test_record_direct_io_operation() {
record_direct_io_operation("read", 4096, true);
record_direct_io_operation("write", 8192, false);
}
#[test]
fn test_update_zero_copy_performance_metrics() {
update_zero_copy_performance_metrics(2, 150.5, 1024 * 1024);
}
#[test]
fn test_metric_names() {
// Verify metric names are defined
assert!(!zero_copy::BUFFER_OPERATIONS_TOTAL.is_empty());
assert!(!zero_copy::MEMORY_COPY_TOTAL.is_empty());
assert!(!zero_copy::THROUGHPUT_MBPS.is_empty());
}
}
+29 -26
View File
@@ -14,41 +14,44 @@
//! Metric name constants for consistent naming across the codebase.
/// Zero-copy operation metric names.
pub mod zero_copy {
/// Total number of zero-copy buffer operations
pub const BUFFER_OPERATIONS_TOTAL: &str = "rustfs_zero_copy_buffer_operations_total";
/// Request-level data plane metric names introduced by ADR 0001.
pub mod data_plane {
/// Total number of selected request paths.
pub const PATH_SELECTED_TOTAL: &str = "rustfs.io.path.selected_total";
/// Total bytes processed by zero-copy buffer operations
pub const BUFFER_BYTES_TOTAL: &str = "rustfs_zero_copy_buffer_bytes_total";
/// Total bytes observed for a given effective copy mode.
pub const COPY_MODE_BYTES_TOTAL: &str = "rustfs.io.copy_mode.bytes_total";
/// Total number of memory copies
pub const MEMORY_COPY_TOTAL: &str = "rustfs_memory_copy_total";
/// Total number of data plane fallbacks.
pub const FALLBACK_TOTAL: &str = "rustfs.io.zero_copy.fallback_total";
/// Total bytes copied in memory
pub const MEMORY_COPY_BYTES_TOTAL: &str = "rustfs_memory_copy_bytes_total";
/// Current active local-disk mmap bytes held by chunk fast paths.
pub const LOCAL_DISK_ACTIVE_MMAP_BYTES: &str = "rustfs.io.local_disk.active_mmap.bytes";
/// Total number of shared reference operations
pub const SHARED_REF_OPERATIONS_TOTAL: &str = "rustfs_shared_ref_operations_total";
/// Total pooled chunks produced or consumed by LocalDisk compatibility paths.
pub const LOCAL_DISK_POOLED_CHUNKS_TOTAL: &str = "rustfs.io.local_disk.pooled_chunks.total";
/// Total number of BufReader layers eliminated
pub const BUFREADER_LAYERS_ELIMINATED_TOTAL: &str = "rustfs_bufreader_layers_eliminated_total";
/// Total pooled bytes produced or consumed by LocalDisk compatibility paths.
pub const LOCAL_DISK_POOLED_BYTES_TOTAL: &str = "rustfs.io.local_disk.pooled_bytes.total";
/// BufReader buffer size distribution
pub const BUFREADER_BUFFER_SIZE_BYTES: &str = "rustfs_bufreader_buffer_size_bytes";
/// Total number of compatibility chunk-stream aggregations performed for LocalDisk reads.
pub const LOCAL_DISK_COMPAT_COLLECT_TOTAL: &str = "rustfs.io.local_disk.compat_collect.total";
/// Total number of Direct I/O operations
pub const DIRECT_IO_OPERATIONS_TOTAL: &str = "rustfs_direct_io_operations_total";
/// Chunk count distribution for LocalDisk compatibility chunk aggregation.
pub const LOCAL_DISK_COMPAT_COLLECT_CHUNKS: &str = "rustfs.io.local_disk.compat_collect.chunks";
/// Total bytes processed by Direct I/O
pub const DIRECT_IO_BYTES_TOTAL: &str = "rustfs_direct_io_bytes_total";
/// Byte distribution for LocalDisk compatibility chunk aggregation.
pub const LOCAL_DISK_COMPAT_COLLECT_BYTES: &str = "rustfs.io.local_disk.compat_collect.bytes";
/// Average copy count per operation
pub const AVG_COPY_COUNT: &str = "rustfs_zero_copy_avg_copy_count";
/// Total number of attempted PUT fast paths.
pub const PUT_FAST_PATH_ATTEMPTS_TOTAL: &str = "rustfs.io.put.fast_path.attempts_total";
/// Throughput in MB/s
pub const THROUGHPUT_MBPS: &str = "rustfs_zero_copy_throughput_mbps";
/// Size distribution for attempted PUT fast paths.
pub const PUT_FAST_PATH_ATTEMPT_SIZE_BYTES: &str = "rustfs.io.put.fast_path.attempt.size.bytes";
/// Memory saved by zero-copy in bytes
pub const MEMORY_SAVED_BYTES: &str = "rustfs_zero_copy_memory_saved_bytes";
/// Total number of transformed PUT selections grouped by transform kind and ingress path.
pub const PUT_TRANSFORM_SELECTED_TOTAL: &str = "rustfs.io.put.transform.selected_total";
/// Size distribution for transformed PUT selections.
pub const PUT_TRANSFORM_SIZE_BYTES: &str = "rustfs.io.put.transform.size.bytes";
}
+37
View File
@@ -0,0 +1,37 @@
[package]
name = "rustfs-object-io"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
homepage.workspace = true
description = "Object I/O policy helpers and zero-copy support primitives for RustFS."
keywords.workspace = true
categories.workspace = true
[lints]
workspace = true
[dependencies]
atoi = { workspace = true }
bytes = { workspace = true }
futures-util = { workspace = true }
rustfs-ecstore = { workspace = true }
rustfs-concurrency = { workspace = true }
rustfs-io-core = { workspace = true }
rustfs-io-metrics = { workspace = true }
rustfs-rio = { workspace = true }
rustfs-s3select-api = { workspace = true }
rustfs-utils = { workspace = true ,features = ["http"]}
http = { workspace = true }
s3s.workspace = true
thiserror = { workspace = true }
time = { workspace = true, features = ["parsing", "formatting"] }
tokio = { workspace = true, features = ["io-util"] }
tokio-util = { workspace = true, features = ["io"] }
astral-tokio-tar = { workspace = true }
uuid = { workspace = true }
[dev-dependencies]
serial_test = { workspace = true }
File diff suppressed because it is too large Load Diff
+16
View File
@@ -0,0 +1,16 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod get;
pub mod put;
File diff suppressed because it is too large Load Diff
+5 -5
View File
@@ -55,7 +55,7 @@ use super::{SwiftError, SwiftResult};
use axum::http::HeaderMap;
use rustfs_credentials::Credentials;
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::store_api::{BucketOperations, BucketOptions, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader};
use rustfs_ecstore::store_api::{BucketOperations, BucketOptions, ChunkNativePutData, ObjectIO, ObjectOperations, ObjectOptions};
use rustfs_rio::HashReader;
use std::collections::HashMap;
use tracing::debug;
@@ -381,8 +381,8 @@ where
let hash_reader = HashReader::from_stream(buf_reader, content_length, content_length, None, None, false)
.map_err(|e| sanitize_storage_error("Hash reader creation", e))?;
// 15. Wrap in PutObjReader as expected by storage layer
let mut put_reader = PutObjReader::new(hash_reader);
// 15. Hand the hash reader to the chunk-native PUT data wrapper
let mut put_reader = ChunkNativePutData::new(hash_reader);
// 16. Upload object to storage
let obj_info = store
@@ -464,8 +464,8 @@ where
let hash_reader = HashReader::from_stream(buf_reader, content_length, content_length, None, None, false)
.map_err(|e| sanitize_storage_error("Hash reader creation", e))?;
// Wrap in PutObjReader
let mut put_reader = PutObjReader::new(hash_reader);
// Hand the hash reader to the chunk-native PUT data wrapper
let mut put_reader = ChunkNativePutData::new(hash_reader);
// Upload object to storage
let obj_info = store
+34 -6
View File
@@ -29,11 +29,21 @@ pub const RUSTFS_MULTIPART_CHECKSUM: &str = "x-rustfs-multipart-checksum";
/// RustFS multipart checksum type metadata key
pub const RUSTFS_MULTIPART_CHECKSUM_TYPE: &str = "x-rustfs-multipart-checksum-type";
const AMZ_CHECKSUM_ALGORITHM: &str = "x-amz-checksum-algorithm";
const AMZ_SDK_CHECKSUM_ALGORITHM: &str = "x-amz-sdk-checksum-algorithm";
/// Checksum type enumeration with flags
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct ChecksumType(pub u32);
impl ChecksumType {
fn algorithm_from_headers(headers: &HeaderMap) -> Option<&str> {
headers
.get(AMZ_CHECKSUM_ALGORITHM)
.and_then(|v| v.to_str().ok())
.or_else(|| headers.get(AMZ_SDK_CHECKSUM_ALGORITHM).and_then(|v| v.to_str().ok()))
}
/// Checksum will be sent in trailing header
pub const TRAILING: ChecksumType = ChecksumType(1 << 0);
@@ -156,10 +166,7 @@ impl ChecksumType {
pub fn from_header(headers: &HeaderMap) -> Self {
Self::from_string_with_obj_type(
headers
.get("x-amz-checksum-algorithm")
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
Self::algorithm_from_headers(headers).unwrap_or(""),
headers.get("x-amz-checksum-type").and_then(|v| v.to_str().ok()).unwrap_or(""),
)
}
@@ -573,7 +580,7 @@ pub fn get_content_checksum(headers: &HeaderMap) -> Result<Option<Checksum>, std
fn get_content_checksum_direct(headers: &HeaderMap) -> (ChecksumType, String) {
let mut checksum_type = ChecksumType::NONE;
if let Some(alg) = headers.get("x-amz-checksum-algorithm").and_then(|v| v.to_str().ok()) {
if let Some(alg) = ChecksumType::algorithm_from_headers(headers) {
checksum_type = ChecksumType::from_string_with_obj_type(
alg,
headers.get("x-amz-checksum-type").and_then(|s| s.to_str().ok()).unwrap_or(""),
@@ -1131,7 +1138,8 @@ fn crc64_combine(poly: u64, crc1: u64, crc2: u64, len2: i64) -> u64 {
#[cfg(test)]
mod tests {
use super::{Checksum, ChecksumType};
use super::{AMZ_SDK_CHECKSUM_ALGORITHM, Checksum, ChecksumType, get_content_checksum_direct};
use http::{HeaderMap, HeaderValue};
#[test]
fn crc64_nvme_add_part_matches_full_object_checksum() {
@@ -1186,4 +1194,24 @@ mod tests {
assert_eq!(combined.encoded, expected.encoded);
assert_eq!(combined.raw, expected.raw);
}
#[test]
fn checksum_type_from_header_supports_sdk_checksum_algorithm_header() {
let mut headers = HeaderMap::new();
headers.insert(AMZ_SDK_CHECKSUM_ALGORITHM, HeaderValue::from_static("CRC32"));
assert_eq!(ChecksumType::from_header(&headers), ChecksumType::CRC32);
}
#[test]
fn get_content_checksum_direct_supports_sdk_checksum_algorithm_header() {
let mut headers = HeaderMap::new();
headers.insert(AMZ_SDK_CHECKSUM_ALGORITHM, HeaderValue::from_static("CRC32"));
headers.insert("x-amz-checksum-crc32", HeaderValue::from_static("nct/nQ=="));
let (checksum_type, checksum_value) = get_content_checksum_direct(&headers);
assert_eq!(checksum_type, ChecksumType::CRC32);
assert_eq!(checksum_value, "nct/nQ==");
}
}
+94
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use crate::compress_index::{Index, TryGetIndex};
use crate::{BlockReadable, BoxReadBlockFuture, Reader};
use pin_project_lite::pin_project;
use rustfs_utils::compress::{CompressionAlgorithm, compress_block, decompress_block};
use rustfs_utils::{put_uvarint, uvarint};
@@ -85,6 +86,31 @@ where
read_buffer: vec![0u8; block_size],
}
}
fn copy_buffered(&mut self, buf: &mut [u8]) -> usize {
if self.pos >= self.buffer.len() || buf.is_empty() {
return 0;
}
let to_copy = min(buf.len(), self.buffer.len() - self.pos);
buf[..to_copy].copy_from_slice(&self.buffer[self.pos..self.pos + to_copy]);
self.pos += to_copy;
if self.pos == self.buffer.len() {
self.buffer.clear();
self.pos = 0;
}
to_copy
}
fn queue_compressed_block(&mut self, uncompressed_data: &[u8]) -> io::Result<()> {
let out = build_compressed_block(uncompressed_data, self.compression_algorithm);
self.written += out.len();
self.uncomp_written += uncompressed_data.len();
self.index.add(self.written as i64, self.uncomp_written as i64)?;
self.buffer = out;
self.pos = 0;
Ok(())
}
}
impl<R> TryGetIndex for CompressReader<R> {
@@ -170,6 +196,50 @@ where
delegate_reader_capabilities_generic_no_index!(CompressReader<R>, inner);
impl<R> BlockReadable for CompressReader<R>
where
R: Reader,
{
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
Box::pin(async move {
if buf.is_empty() {
return Ok(0);
}
let mut written = self.copy_buffered(buf);
while written < buf.len() {
if self.done {
break;
}
self.temp_buffer.resize(self.block_size, 0);
let n = {
let inner = &mut self.inner;
let temp = &mut self.temp_buffer[..self.block_size];
match inner.read_block(temp).await {
Ok(n) => n,
Err(err) if err.kind() == io::ErrorKind::UnexpectedEof => 0,
Err(err) => return Err(err),
}
};
if n == 0 {
self.done = true;
self.temp_buffer.clear();
break;
}
let block = self.temp_buffer[..n].to_vec();
self.temp_buffer.clear();
self.queue_compressed_block(&block)?;
written += self.copy_buffered(&mut buf[written..]);
}
Ok(written)
})
}
}
pin_project! {
/// A reader wrapper that decompresses data on the fly using DEFLATE algorithm.
/// Header format:
@@ -390,6 +460,7 @@ fn build_compressed_block(uncompressed_data: &[u8], compression_algorithm: Compr
#[cfg(test)]
mod tests {
use super::*;
use crate::{BlockReadable, WarpReader};
use rand::RngExt;
use std::io::Cursor;
use tokio::io::{AsyncReadExt, BufReader};
@@ -479,4 +550,27 @@ mod tests {
assert_eq!(&decompressed, &data);
}
#[tokio::test]
async fn test_compress_reader_read_block_round_trips() {
let data = b"hello world, hello world, hello world!";
let reader = Cursor::new(data.to_vec());
let mut compress_reader = CompressReader::new(WarpReader::new(reader), CompressionAlgorithm::Gzip);
let mut compressed = Vec::new();
let mut buf = [0u8; 19];
loop {
let n = compress_reader.read_block(&mut buf).await.unwrap();
if n == 0 {
break;
}
compressed.extend_from_slice(&buf[..n]);
}
let mut decompress_reader = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::Gzip);
let mut decompressed = Vec::new();
decompress_reader.read_to_end(&mut decompressed).await.unwrap();
assert_eq!(&decompressed, data);
}
}
+139 -43
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use crate::compress_index::{Index, TryGetIndex};
use crate::{BlockReadable, BoxReadBlockFuture, Reader};
use aes_gcm::aead::Aead;
use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
use pin_project_lite::pin_project;
@@ -59,6 +60,69 @@ where
}
}
fn encrypt_segment_bytes(cipher: &Aes256Gcm, nonce_bytes: &[u8; 12], plaintext: &[u8]) -> std::io::Result<Vec<u8>> {
let nonce = Nonce::try_from(nonce_bytes.as_slice()).map_err(|_| Error::other("invalid nonce length"))?;
let plaintext_len = plaintext.len();
let crc = {
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
hasher.update(plaintext);
hasher.finalize() as u32
};
let ciphertext = cipher
.encrypt(&nonce, plaintext)
.map_err(|e| Error::other(format!("encrypt error: {e}")))?;
let int_len = put_uvarint_len(plaintext_len as u64);
let clen = int_len + ciphertext.len() + 4;
let mut header = [0u8; 8];
header[0] = 0x00;
header[1] = (clen & 0xFF) as u8;
header[2] = ((clen >> 8) & 0xFF) as u8;
header[3] = ((clen >> 16) & 0xFF) as u8;
header[4] = (crc & 0xFF) as u8;
header[5] = ((crc >> 8) & 0xFF) as u8;
header[6] = ((crc >> 16) & 0xFF) as u8;
header[7] = ((crc >> 24) & 0xFF) as u8;
debug!(
"encrypt block header typ=0 len={} header={:?} plaintext_len={} ciphertext_len={}",
clen,
header,
plaintext_len,
ciphertext.len()
);
let mut out = Vec::with_capacity(8 + int_len + ciphertext.len());
out.extend_from_slice(&header);
let mut plaintext_len_buf = vec![0u8; int_len];
put_uvarint(&mut plaintext_len_buf, plaintext_len as u64);
out.extend_from_slice(&plaintext_len_buf);
out.extend_from_slice(&ciphertext);
Ok(out)
}
impl<R> EncryptReader<R>
where
R: Reader,
{
fn copy_buffered(&mut self, buf: &mut [u8]) -> usize {
if self.buffer_pos >= self.buffer.len() || buf.is_empty() {
return 0;
}
let to_copy = buf.len().min(self.buffer.len() - self.buffer_pos);
buf[..to_copy].copy_from_slice(&self.buffer[self.buffer_pos..self.buffer_pos + to_copy]);
self.buffer_pos += to_copy;
if self.buffer_pos == self.buffer.len() {
self.buffer.clear();
self.buffer_pos = 0;
}
to_copy
}
fn encrypt_segment(&self, plaintext: &[u8]) -> std::io::Result<Vec<u8>> {
let nonce = derive_block_nonce(&self.base_nonce, self.block_index);
encrypt_segment_bytes(&self.cipher, &nonce, plaintext)
}
}
impl<R> AsyncRead for EncryptReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
@@ -97,49 +161,8 @@ where
*this.buffer_pos += to_copy;
Poll::Ready(Ok(()))
} else {
// Encrypt the chunk
let block_nonce = derive_block_nonce(this.base_nonce, *this.block_index);
let nonce = Nonce::try_from(block_nonce.as_slice()).map_err(|_| Error::other("invalid nonce length"))?;
let plaintext = &this.read_buffer[..n];
let plaintext_len = plaintext.len();
let crc = {
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
hasher.update(plaintext);
hasher.finalize() as u32
};
let ciphertext = this
.cipher
.encrypt(&nonce, plaintext)
.map_err(|e| Error::other(format!("encrypt error: {e}")))?;
let int_len = put_uvarint_len(plaintext_len as u64);
let clen = int_len + ciphertext.len() + 4;
// Header: 8 bytes
// 0: type (0 = encrypted, 0xFF = end)
// 1-3: length (little endian u24, ciphertext length)
// 4-7: CRC32 of ciphertext (little endian u32)
let mut header = [0u8; 8];
header[0] = 0x00; // 0 = encrypted
header[1] = (clen & 0xFF) as u8;
header[2] = ((clen >> 8) & 0xFF) as u8;
header[3] = ((clen >> 16) & 0xFF) as u8;
header[4] = (crc & 0xFF) as u8;
header[5] = ((crc >> 8) & 0xFF) as u8;
header[6] = ((crc >> 16) & 0xFF) as u8;
header[7] = ((crc >> 24) & 0xFF) as u8;
debug!(
"encrypt block header typ=0 len={} header={:?} plaintext_len={} ciphertext_len={}",
clen,
header,
plaintext_len,
ciphertext.len()
);
let mut out = Vec::with_capacity(8 + int_len + ciphertext.len());
out.extend_from_slice(&header);
let mut plaintext_len_buf = [0u8; 10];
let encoded_len = put_uvarint(&mut plaintext_len_buf, plaintext_len as u64);
out.extend_from_slice(&plaintext_len_buf[..encoded_len]);
out.extend_from_slice(&ciphertext);
*this.buffer = out;
*this.buffer = encrypt_segment_bytes(this.cipher, &block_nonce, &this.read_buffer[..n])?;
*this.buffer_pos = 0;
*this.block_index += 1;
let to_copy = std::cmp::min(buf.remaining(), this.buffer.len());
@@ -164,6 +187,50 @@ where
}
}
impl<R> BlockReadable for EncryptReader<R>
where
R: Reader,
{
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
Box::pin(async move {
if buf.is_empty() {
return Ok(0);
}
let mut written = self.copy_buffered(buf);
while written < buf.len() {
if self.finished {
break;
}
let mut plaintext = vec![0u8; 8 * 1024];
let n = match self.inner.read_block(&mut plaintext).await {
Ok(n) => n,
Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => 0,
Err(err) => return Err(err),
};
if n == 0 {
self.buffer = [0xFF, 0, 0, 0, 0, 0, 0, 0].to_vec();
self.buffer_pos = 0;
self.finished = true;
} else {
self.buffer = self.encrypt_segment(&plaintext[..n])?;
self.buffer_pos = 0;
}
let copied = self.copy_buffered(&mut buf[written..]);
written += copied;
if copied == 0 && self.finished {
break;
}
}
Ok(written)
})
}
}
pin_project! {
/// A reader wrapper that decrypts data on the fly using AES-256-GCM.
/// This is a demonstration. For production, use a secure and audited crypto library.
@@ -486,7 +553,7 @@ mod tests {
use std::pin::Pin;
use std::task::{Context, Poll};
use crate::HardLimitReader;
use crate::{BlockReadable, HardLimitReader, WarpReader};
use super::*;
use futures::StreamExt;
@@ -674,6 +741,35 @@ mod tests {
assert_eq!(&decrypted, &data);
}
#[tokio::test]
async fn test_encrypt_reader_read_block_round_trips() {
let data = b"hello sse encrypt via blocks";
let mut key = [0u8; 32];
let mut nonce = [0u8; 12];
rand::rng().fill_bytes(&mut key);
rand::rng().fill_bytes(&mut nonce);
let reader = Cursor::new(data.to_vec());
let mut encrypt_reader = EncryptReader::new(WarpReader::new(reader), key, nonce);
let mut encrypted = Vec::new();
let mut buf = [0u8; 17];
loop {
let n = encrypt_reader.read_block(&mut buf).await.unwrap();
if n == 0 {
break;
}
encrypted.extend_from_slice(&buf[..n]);
}
let reader = Cursor::new(encrypted);
let mut decrypt_reader = DecryptReader::new(WarpReader::new(reader), key, nonce);
let mut decrypted = Vec::new();
decrypt_reader.read_to_end(&mut decrypted).await.unwrap();
assert_eq!(&decrypted, data);
}
#[tokio::test]
async fn test_decrypt_reader_large_with_small_chunks() {
let size = 1024 * 1024;
+51 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::compress_index::{Index, TryGetIndex};
use crate::{EtagResolvable, HashReaderDetector, HashReaderMut};
use crate::{BlockReadable, BoxReadBlockFuture, EtagResolvable, HashReaderDetector, HashReaderMut, Reader};
use md5::{Digest, Md5};
use pin_project_lite::pin_project;
use std::pin::Pin;
@@ -135,9 +135,41 @@ where
}
}
impl<R> BlockReadable for EtagReader<R>
where
R: Reader,
{
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
Box::pin(async move {
let n = match self.inner.read_block(buf).await {
Ok(n) => n,
Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => 0,
Err(err) => return Err(err),
};
if n > 0 {
self.md5.update(&buf[..n]);
return Ok(n);
}
self.finished = true;
if let Some(checksum) = &self.checksum {
let etag = self.md5.clone().finalize().to_vec();
let etag_hex = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
if checksum != &etag_hex {
error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag_hex);
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Checksum mismatch"));
}
}
Ok(0)
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{BlockReadable, WarpReader};
use rand::RngExt;
use std::io::Cursor;
use tokio::io::{AsyncReadExt, BufReader};
@@ -180,6 +212,24 @@ mod tests {
assert_eq!(etag, Some(expected));
}
#[tokio::test]
async fn test_etag_reader_read_block_updates_checksum() {
let data = b"hello world";
let mut hasher = Md5::new();
hasher.update(data);
let expected = faster_hex::hex_string(hasher.finalize().as_slice()).to_string();
let reader = BufReader::new(&data[..]);
let reader = Box::new(WarpReader::new(reader));
let mut etag_reader = EtagReader::new(reader, Some(expected.clone()));
let mut buf = [0_u8; 32];
let n = etag_reader.read_block(&mut buf).await.unwrap();
assert_eq!(n, data.len());
assert_eq!(&buf[..n], data);
assert_eq!(etag_reader.read_block(&mut buf).await.unwrap(), 0);
assert_eq!(etag_reader.try_resolve_etag(), Some(expected));
}
#[tokio::test]
async fn test_etag_reader_multiple_get() {
let data = b"abc123";
+57
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{BlockReadable, BoxReadBlockFuture, Reader};
use pin_project_lite::pin_project;
use std::io::{Error, Result};
use std::pin::Pin;
@@ -61,11 +62,51 @@ where
delegate_reader_capabilities_generic!(HardLimitReader<R>, inner);
impl<R> BlockReadable for HardLimitReader<R>
where
R: Reader,
{
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
Box::pin(async move {
if self.remaining < 0 {
return Err(Error::other("input provided more bytes than specified"));
}
let max_len = match usize::try_from(self.remaining) {
Ok(remaining) => remaining.min(buf.len()),
Err(_) => buf.len(),
};
if max_len == 0 {
let mut probe = [0_u8; 1];
match self.inner.read_block(&mut probe).await {
Ok(0) => return Ok(0),
Ok(n) => {
self.remaining -= n as i64;
return Err(Error::other("input provided more bytes than specified"));
}
Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(0),
Err(err) => return Err(err),
}
}
let n = self.inner.read_block(&mut buf[..max_len]).await?;
self.remaining -= n as i64;
if self.remaining < 0 {
return Err(Error::other("input provided more bytes than specified"));
}
Ok(n)
})
}
}
#[cfg(test)]
mod tests {
use std::vec;
use super::*;
use crate::{BlockReadable, WarpReader};
use rustfs_utils::read_full;
use tokio::io::{AsyncReadExt, BufReader};
@@ -128,4 +169,20 @@ mod tests {
assert_eq!(n, 0);
assert_eq!(&buf, data);
}
#[tokio::test]
async fn test_hardlimit_reader_read_block_enforces_limit() {
let data = b"abcdef";
let reader = BufReader::new(&data[..]);
let reader = Box::new(WarpReader::new(reader));
let mut hardlimit = HardLimitReader::new(reader, 3);
let mut buf = [0_u8; 8];
let n = hardlimit.read_block(&mut buf).await.unwrap();
assert_eq!(n, 3);
assert_eq!(&buf[..n], b"abc");
let err = hardlimit.read_block(&mut buf).await.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::Other);
}
}
+179 -72
View File
@@ -90,7 +90,10 @@ use crate::ChecksumType;
use crate::Sha256Hasher;
use crate::compress_index::{Index, TryGetIndex};
use crate::get_content_checksum;
use crate::{DynReader, EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, WarpReader, boxed_reader, wrap_reader};
use crate::{
BlockReadable, BoxReadBlockFuture, DynReader, EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, WarpReader,
boxed_reader, wrap_reader,
};
use base64::Engine;
use base64::engine::general_purpose;
use http::HeaderMap;
@@ -408,6 +411,23 @@ impl HashReader {
Ok(())
}
pub fn enable_auto_checksum(&mut self, checksum_type: ChecksumType) -> Result<(), std::io::Error> {
if !checksum_type.is_set() {
return Ok(());
}
let Some(hasher) = checksum_type.hasher() else {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid checksum type"));
};
self.content_hash = Some(Checksum {
checksum_type,
..Default::default()
});
self.content_hasher = Some(hasher);
Ok(())
}
pub fn checksum(&self) -> Option<Checksum> {
if self
.content_hash
@@ -449,6 +469,97 @@ impl HashReader {
}
map
}
pub fn finalize_content_hash(&mut self) -> std::io::Result<Option<Checksum>> {
self.finish_checksum_validation()?;
Ok(self.content_hash.clone())
}
fn update_read_state(&mut self, data: &[u8]) -> std::io::Result<()> {
self.bytes_read += data.len() as u64;
if data.is_empty() {
return Ok(());
}
if let Some(hasher) = self.content_sha256_hasher.as_mut() {
hasher.write_all(data)?;
}
if let Some(hasher) = self.content_hasher.as_mut() {
hasher.write_all(data)?;
}
Ok(())
}
fn finish_checksum_validation(&mut self) -> std::io::Result<()> {
if self.checksum_on_finish {
return Ok(());
}
if let (Some(mut hasher), Some(expected_sha256)) = (self.content_sha256_hasher.take(), self.content_sha256.as_ref()) {
let sha256 = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower);
if sha256 != *expected_sha256 {
error!("SHA256 mismatch, expected={:?}, actual={:?}", expected_sha256, sha256);
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "SHA256 mismatch"));
}
}
if let Some(mut expected_content_hash) = self.content_hash.clone()
&& let Some(mut hasher) = self.content_hasher.take()
{
if expected_content_hash.checksum_type.trailing()
&& let Some(trailer) = self.trailer_s3s.as_ref()
&& let Some(Some(checksum_str)) = trailer.read(|headers| {
expected_content_hash
.checksum_type
.key()
.and_then(|key| headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string())))
})
{
expected_content_hash.encoded = checksum_str;
expected_content_hash.raw = general_purpose::STANDARD
.decode(&expected_content_hash.encoded)
.map_err(|_| std::io::Error::other("Invalid base64 checksum"))?;
if expected_content_hash.raw.is_empty() {
return Err(std::io::Error::other("Content hash mismatch"));
}
}
let content_hash = hasher.finalize();
if expected_content_hash.encoded.is_empty() {
expected_content_hash.raw = content_hash.clone();
expected_content_hash.encoded = general_purpose::STANDARD.encode(&content_hash);
self.content_hash = Some(expected_content_hash);
} else if content_hash != expected_content_hash.raw {
let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower);
let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower);
error!(
"Content hash mismatch, type={:?}, encoded={:?}, expected={:?}, actual={:?}",
expected_content_hash.checksum_type, expected_content_hash.encoded, expected_hex, actual_hex
);
let checksum_err = crate::errors::ChecksumMismatch {
want: expected_hex,
got: actual_hex,
};
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, checksum_err));
}
}
self.checksum_on_finish = true;
Ok(())
}
pub async fn read_block(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.inner.read_block(buf).await?;
self.update_read_state(&buf[..n])?;
if n == 0 {
self.finish_checksum_validation()?;
}
Ok(n)
}
}
impl HashReaderMut for HashReader {
@@ -508,84 +619,23 @@ impl HashReaderMut for HashReader {
impl AsyncRead for HashReader {
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
let this = self.project();
let this = self.get_mut();
let before = buf.filled().len();
match this.inner.poll_read(cx, buf) {
match Pin::new(&mut this.inner).poll_read(cx, buf) {
Poll::Pending => Poll::Pending,
Poll::Ready(Ok(())) => {
let data = &buf.filled()[before..];
let filled = data.len();
*this.bytes_read += filled as u64;
if filled > 0 {
// Update SHA256 hasher
if let Some(hasher) = this.content_sha256_hasher
&& let Err(e) = hasher.write_all(data)
{
error!("SHA256 hasher write error, error={:?}", e);
return Poll::Ready(Err(std::io::Error::other(e)));
}
// Update content hasher
if let Some(hasher) = this.content_hasher
&& let Err(e) = hasher.write_all(data)
{
return Poll::Ready(Err(std::io::Error::other(e)));
}
if let Err(e) = this.update_read_state(data) {
error!("hash reader state update error, error={:?}", e);
return Poll::Ready(Err(std::io::Error::other(e)));
}
if filled == 0 && !*this.checksum_on_finish {
// check SHA256
if let (Some(hasher), Some(expected_sha256)) = (this.content_sha256_hasher, this.content_sha256) {
let sha256 = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower);
if sha256 != *expected_sha256 {
error!("SHA256 mismatch, expected={:?}, actual={:?}", expected_sha256, sha256);
return Poll::Ready(Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "SHA256 mismatch")));
}
}
// check content hasher
if let (Some(hasher), Some(expected_content_hash)) = (this.content_hasher, this.content_hash) {
if expected_content_hash.checksum_type.trailing()
&& let Some(trailer) = this.trailer_s3s.as_ref()
&& let Some(Some(checksum_str)) = trailer.read(|headers| {
expected_content_hash
.checksum_type
.key()
.and_then(|key| headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string())))
})
{
expected_content_hash.encoded = checksum_str;
expected_content_hash.raw = general_purpose::STANDARD
.decode(&expected_content_hash.encoded)
.map_err(|_| std::io::Error::other("Invalid base64 checksum"))?;
if expected_content_hash.raw.is_empty() {
return Poll::Ready(Err(std::io::Error::other("Content hash mismatch")));
}
}
let content_hash = hasher.finalize();
if content_hash != expected_content_hash.raw {
let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower);
let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower);
error!(
"Content hash mismatch, type={:?}, encoded={:?}, expected={:?}, actual={:?}",
expected_content_hash.checksum_type, expected_content_hash.encoded, expected_hex, actual_hex
);
// Use ChecksumMismatch error so that API layer can return BadDigest
let checksum_err = crate::errors::ChecksumMismatch {
want: expected_hex,
got: actual_hex,
};
return Poll::Ready(Err(std::io::Error::new(std::io::ErrorKind::InvalidData, checksum_err)));
}
}
*this.checksum_on_finish = true;
if filled == 0
&& let Err(e) = this.finish_checksum_validation()
{
return Poll::Ready(Err(e));
}
Poll::Ready(Ok(()))
}
@@ -623,13 +673,39 @@ impl TryGetIndex for HashReader {
}
}
impl BlockReadable for HashReader {
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
Box::pin(async move { self.read_block(buf).await })
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{DecryptReader, EncryptReader, encrypt_reader, wrap_reader};
use rand::RngExt;
use std::io::Cursor;
use tokio::io::{AsyncReadExt, BufReader};
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf};
struct UnexpectedEofReader;
impl AsyncRead for UnexpectedEofReader {
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
}
impl BlockReadable for UnexpectedEofReader {
fn read_block<'a>(&'a mut self, _buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
Box::pin(async { Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "synthetic unexpected eof")) })
}
}
impl EtagResolvable for UnexpectedEofReader {}
impl HashReaderDetector for UnexpectedEofReader {}
impl TryGetIndex for UnexpectedEofReader {}
#[tokio::test]
async fn test_hashreader_wrapping_logic() {
@@ -742,6 +818,37 @@ mod tests {
assert_eq!(buf, data);
}
#[tokio::test]
async fn test_hashreader_read_block_reads_full_and_tail_block() {
let data = b"hello block reader";
let reader = BufReader::new(Cursor::new(&data[..]));
let reader = Box::new(WarpReader::new(reader));
let mut hash_reader = HashReader::new(reader, data.len() as i64, data.len() as i64, None, None, false).unwrap();
let mut first = [0_u8; 8];
let n1 = hash_reader.read_block(&mut first).await.unwrap();
assert_eq!(n1, 8);
assert_eq!(&first[..n1], b"hello bl");
let mut second = [0_u8; 32];
let n2 = hash_reader.read_block(&mut second).await.unwrap();
assert_eq!(n2, data.len() - n1);
assert_eq!(&second[..n2], b"ock reader");
let n3 = hash_reader.read_block(&mut second).await.unwrap();
assert_eq!(n3, 0);
}
#[tokio::test]
async fn test_hashreader_read_block_propagates_unexpected_eof() {
let mut hash_reader = HashReader::new(Box::new(UnexpectedEofReader), 0, 0, None, None, true).unwrap();
let mut buf = [0_u8; 8];
let err = hash_reader.read_block(&mut buf).await.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
}
#[tokio::test]
async fn test_hashreader_new_logic() {
let data = b"test data";
+68 -36
View File
@@ -23,7 +23,6 @@ use rustfs_utils::get_env_opt_str;
use std::io::IoSlice;
use std::io::{self, Error};
use std::net::IpAddr;
use std::ops::Not as _;
use std::pin::Pin;
use std::sync::LazyLock;
use std::task::{Context, Poll};
@@ -137,6 +136,71 @@ fn get_http_client(url: &str) -> Client {
CLIENT.clone()
}
type HttpByteStream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>> + Send + Sync>>;
async fn request_http_byte_stream(
url: String,
method: Method,
headers: HeaderMap,
body: Option<Vec<u8>>,
meter_stream_recv_bytes: bool,
) -> io::Result<(bool, HttpByteStream)> {
let track_internode_metrics = is_internode_rpc_url(&url);
let client = get_http_client(&url);
let mut request: RequestBuilder = client.request(method, url.clone()).headers(headers);
if let Some(body) = body {
request = request.body(body);
}
let resp = request.send().await.map_err(|e| {
if track_internode_metrics {
global_internode_metrics().record_error();
}
Error::other(format!("HttpReader HTTP request error: {e}"))
})?;
if !resp.status().is_success() {
if track_internode_metrics {
global_internode_metrics().record_error();
}
return Err(Error::other(format!(
"HttpReader HTTP request failed with non-200 status {}",
resp.status()
)));
}
if track_internode_metrics {
global_internode_metrics().record_outgoing_request();
}
let stream = resp
.bytes_stream()
.map_ok(move |bytes| {
if track_internode_metrics && meter_stream_recv_bytes {
global_internode_metrics().record_recv_bytes(bytes.len());
}
bytes
})
.map_err(move |e| {
if track_internode_metrics {
global_internode_metrics().record_error();
}
Error::other(format!("HttpReader stream error: {e}"))
});
Ok((track_internode_metrics, Box::pin(stream)))
}
pub async fn open_http_byte_stream(
url: String,
method: Method,
headers: HeaderMap,
body: Option<Vec<u8>>,
) -> io::Result<HttpByteStream> {
let (_track_internode_metrics, stream) = request_http_byte_stream(url, method, headers, body, true).await?;
Ok(stream)
}
pin_project! {
pub struct HttpReader {
url:String,
@@ -161,43 +225,11 @@ impl HttpReader {
body: Option<Vec<u8>>,
_read_buf_size: usize,
) -> io::Result<Self> {
let track_internode_metrics = is_internode_rpc_url(&url);
let client = get_http_client(&url);
let mut request: RequestBuilder = client.request(method.clone(), url.clone()).headers(headers.clone());
if let Some(body) = body {
request = request.body(body);
}
let resp = request.send().await.map_err(|e| {
if track_internode_metrics {
global_internode_metrics().record_error();
}
Error::other(format!("HttpReader HTTP request error: {e}"))
})?;
if resp.status().is_success().not() {
if track_internode_metrics {
global_internode_metrics().record_error();
}
return Err(Error::other(format!(
"HttpReader HTTP request failed with non-200 status {}",
resp.status()
)));
}
if track_internode_metrics {
global_internode_metrics().record_outgoing_request();
}
let stream = resp.bytes_stream().map_err(move |e| {
if track_internode_metrics {
global_internode_metrics().record_error();
}
Error::other(format!("HttpReader stream error: {e}"))
});
let (track_internode_metrics, stream) =
request_http_byte_stream(url.clone(), method.clone(), headers.clone(), body, false).await?;
Ok(Self {
inner: StreamReader::new(Box::pin(stream)),
inner: StreamReader::new(stream),
url,
method,
headers,
+138 -2
View File
@@ -15,6 +15,10 @@
// Default encryption block size - aligned with system default read buffer size (1MB)
pub const DEFAULT_ENCRYPTION_BLOCK_SIZE: usize = 1024 * 1024;
use std::future::Future;
use std::pin::Pin;
use tokio::io::AsyncReadExt;
macro_rules! delegate_reader_capabilities_generic {
($name:ident<$inner_ty:ident>, $inner:ident) => {
impl<$inner_ty> crate::EtagResolvable for $name<$inner_ty>
@@ -114,14 +118,39 @@ pub use compress_index::{Index, TryGetIndex};
mod etag;
pub type BoxReadBlockFuture<'a> = Pin<Box<dyn Future<Output = std::io::Result<usize>> + Send + 'a>>;
pub trait BlockReadable {
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a>;
}
fn read_block_via_async_read<'a, R>(reader: &'a mut R, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a>
where
R: tokio::io::AsyncRead + Unpin + Send + Sync + 'a,
{
Box::pin(async move {
let mut total = 0;
while total < buf.len() {
match reader.read(&mut buf[total..]).await {
Ok(0) => return Ok(total),
Ok(n) => total += n,
Err(err) => return Err(err),
}
}
Ok(total)
})
}
pub trait ReadStream: tokio::io::AsyncRead + Unpin + Send + Sync {}
impl<T> ReadStream for T where T: tokio::io::AsyncRead + Unpin + Send + Sync {}
pub trait ReaderCapabilities: EtagResolvable + HashReaderDetector + TryGetIndex {}
impl<T> ReaderCapabilities for T where T: EtagResolvable + HashReaderDetector + TryGetIndex {}
pub trait Reader: ReadStream + ReaderCapabilities {}
impl<T> Reader for T where T: ReadStream + ReaderCapabilities {}
pub trait Reader: ReadStream + ReaderCapabilities + BlockReadable {}
impl<T> Reader for T where T: ReadStream + ReaderCapabilities + BlockReadable {}
pub type DynReader = Box<dyn Reader>;
@@ -154,6 +183,42 @@ pub trait HashReaderDetector {
}
}
impl<R> BlockReadable for crate::WarpReader<R>
where
R: tokio::io::AsyncRead + Unpin + Send + Sync,
{
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
read_block_via_async_read(self, buf)
}
}
impl<R> BlockReadable for tokio::io::BufReader<R>
where
R: tokio::io::AsyncRead + Unpin + Send + Sync,
{
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
read_block_via_async_read(self, buf)
}
}
impl<R> BlockReadable for crate::LimitReader<R>
where
R: Reader,
{
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
read_block_via_async_read(self, buf)
}
}
impl<R> BlockReadable for crate::DecryptReader<R>
where
R: Reader,
{
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
read_block_via_async_read(self, buf)
}
}
pub fn boxed_reader<R>(reader: R) -> DynReader
where
R: Reader + 'static,
@@ -198,3 +263,74 @@ where
self.as_ref().try_get_index()
}
}
impl<T> BlockReadable for Box<T>
where
T: BlockReadable + ?Sized,
{
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
self.as_mut().read_block(buf)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::VecDeque;
use std::io::{self, ErrorKind};
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
enum ReadStep {
Data(Vec<u8>),
Error(ErrorKind),
Eof,
}
struct StepReader {
steps: VecDeque<ReadStep>,
}
impl StepReader {
fn new(steps: impl IntoIterator<Item = ReadStep>) -> Self {
Self {
steps: steps.into_iter().collect(),
}
}
}
impl AsyncRead for StepReader {
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
match self.steps.pop_front().unwrap_or(ReadStep::Eof) {
ReadStep::Data(data) => {
buf.put_slice(&data);
Poll::Ready(Ok(()))
}
ReadStep::Error(kind) => Poll::Ready(Err(io::Error::new(kind, "synthetic read failure"))),
ReadStep::Eof => Poll::Ready(Ok(())),
}
}
}
#[tokio::test]
async fn test_read_block_via_async_read_preserves_midstream_error_kind() {
let reader = StepReader::new([ReadStep::Data(b"ab".to_vec()), ReadStep::Error(ErrorKind::ConnectionReset)]);
let mut reader = WarpReader::new(reader);
let mut buf = [0_u8; 4];
let err = reader.read_block(&mut buf).await.unwrap_err();
assert_eq!(err.kind(), ErrorKind::ConnectionReset);
}
#[tokio::test]
async fn test_read_block_via_async_read_returns_zero_on_initial_eof() {
let mut reader = WarpReader::new(StepReader::new([ReadStep::Eof]));
let mut buf = [0_u8; 4];
let n = reader.read_block(&mut buf).await.unwrap();
assert_eq!(n, 0);
}
}
@@ -24,7 +24,7 @@ use rustfs_ecstore::{
pools::path2_bucket_object_with_base_path,
store::ECStore,
store_api::{
BucketOperations, MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader,
BucketOperations, ChunkNativePutData, MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions,
},
tier::{
tier_config::{TierConfig, TierMinIO, TierType},
@@ -235,7 +235,7 @@ async fn create_test_lock_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
/// Test helper: Upload test object
async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) {
let mut reader = PutObjReader::from_vec(data.to_vec());
let mut reader = ChunkNativePutData::from_vec(data.to_vec());
let object_info = (**ecstore)
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
@@ -781,7 +781,7 @@ mod serial_tests {
.await
.expect("Failed to set lifecycle configuration");
let mut reader = PutObjReader::from_vec(put_payload.to_vec());
let mut reader = ChunkNativePutData::from_vec(put_payload.to_vec());
let mut metadata = HashMap::new();
metadata.insert("content-type".to_string(), "text/plain".to_string());
ecstore
@@ -838,7 +838,7 @@ mod serial_tests {
.expect("Failed to create multipart upload");
let part_data = b"multipart immediate transition";
let mut reader = PutObjReader::from_vec(part_data.to_vec());
let mut reader = ChunkNativePutData::from_vec(part_data.to_vec());
let part = ecstore
.put_object_part(
multipart_bucket.as_str(),
@@ -903,7 +903,7 @@ mod serial_tests {
.get_object_info(src_bucket.as_str(), src_object, &ObjectOptions::default())
.await
.expect("Failed to load source object info");
src_info.put_object_reader = Some(PutObjReader::from_vec(payload.to_vec()));
src_info.put_object_reader = Some(ChunkNativePutData::from_vec(payload.to_vec()));
ecstore
.copy_object(
@@ -969,7 +969,7 @@ mod serial_tests {
.await
.expect("Failed to create multipart upload");
let mut part1_reader = PutObjReader::from_vec(part1);
let mut part1_reader = ChunkNativePutData::from_vec(part1);
let uploaded_part1 = ecstore
.put_object_part(
bucket_name.as_str(),
@@ -982,7 +982,7 @@ mod serial_tests {
.await
.expect("Failed to upload first multipart part");
let mut part2_reader = PutObjReader::from_vec(part2);
let mut part2_reader = ChunkNativePutData::from_vec(part2);
let uploaded_part2 = ecstore
.put_object_part(
bucket_name.as_str(),