From 642d83f0e48feadccf32c4b7682308d203df4b3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Wed, 15 Apr 2026 10:54:41 +0800 Subject: [PATCH] revert: remove #2351 chunk I/O and object-io crate (phase 7) (#2543) Co-authored-by: houseme --- Cargo.lock | 26 - Cargo.toml | 2 - crates/ecstore/src/config/storageclass.rs | 51 +- crates/ecstore/src/data_movement.rs | 8 +- crates/ecstore/src/set_disk.rs | 183 +- crates/ecstore/src/store_api/types.rs | 14 - crates/io-core/Cargo.toml | 2 - crates/io-core/src/adapter.rs | 124 - crates/io-core/src/chunk.rs | 276 --- crates/io-core/src/lib.rs | 4 - crates/io-core/src/pool.rs | 42 +- crates/io-metrics/src/lib.rs | 626 ++--- crates/io-metrics/src/metric_names.rs | 55 +- crates/object-io/Cargo.toml | 36 - crates/object-io/src/get.rs | 804 ------- crates/object-io/src/lib.rs | 16 - crates/object-io/src/put.rs | 1439 ----------- crates/rio/src/hash_reader.rs | 27 +- rustfs/Cargo.toml | 1 - rustfs/src/app/multipart_usecase.rs | 165 +- rustfs/src/app/object_usecase.rs | 2653 ++++++++++++++++++--- rustfs/src/error.rs | 18 +- 22 files changed, 2757 insertions(+), 3815 deletions(-) delete mode 100644 crates/io-core/src/adapter.rs delete mode 100644 crates/io-core/src/chunk.rs delete mode 100644 crates/object-io/Cargo.toml delete mode 100644 crates/object-io/src/get.rs delete mode 100644 crates/object-io/src/lib.rs delete mode 100644 crates/object-io/src/put.rs diff --git a/Cargo.lock b/Cargo.lock index c8b42ce34..fe59d3d53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7730,7 +7730,6 @@ dependencies = [ "rustfs-metrics", "rustfs-notify", "rustfs-object-capacity", - "rustfs-object-io", "rustfs-obs", "rustfs-policy", "rustfs-protocols", @@ -8074,8 +8073,6 @@ name = "rustfs-io-core" version = "0.0.5" dependencies = [ "bytes", - "futures-core", - "futures-util", "memmap2 0.9.10", "rustfs-io-metrics", "thiserror 2.0.18", @@ -8264,29 +8261,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "rustfs-object-io" -version = "0.0.5" -dependencies = [ - "astral-tokio-tar", - "atoi", - "bytes", - "futures-util", - "http 1.4.0", - "rustfs-ecstore", - "rustfs-io-core", - "rustfs-io-metrics", - "rustfs-rio", - "rustfs-s3select-api", - "rustfs-utils", - "s3s", - "thiserror 2.0.18", - "time", - "tokio", - "tokio-util", - "uuid", -] - [[package]] name = "rustfs-obs" version = "0.0.5" diff --git a/Cargo.toml b/Cargo.toml index ef6382de0..5c995a468 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,6 @@ members = [ "crates/workers", # Worker thread pools and task scheduling "crates/io-metrics", # Zero-copy metrics collection for performance analysis "crates/io-core", # Zero-copy core reader and writer implementations - "crates/object-io", # Object I/O policy and zero-copy helper primitives "crates/zip", # ZIP file handling and compression ] resolver = "3" @@ -99,7 +98,6 @@ rustfs-metrics = { path = "crates/metrics", version = "0.0.5" } rustfs-notify = { path = "crates/notify", version = "0.0.5" } rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" } rustfs-io-core = { path = "crates/io-core", version = "0.0.5" } -rustfs-object-io = { path = "crates/object-io", version = "0.0.5" } rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" } rustfs-obs = { path = "crates/obs", version = "0.0.5" } rustfs-policy = { path = "crates/policy", version = "0.0.5" } diff --git a/crates/ecstore/src/config/storageclass.rs b/crates/ecstore/src/config/storageclass.rs index 2f6f92147..5fe70f4a0 100644 --- a/crates/ecstore/src/config/storageclass.rs +++ b/crates/ecstore/src/config/storageclass.rs @@ -150,7 +150,18 @@ impl Config { return false; } - shard_size as usize <= self.inline_shard_limit_bytes(versioned) + let shard_size = shard_size as usize; + + let mut inline_block = DEFAULT_INLINE_BLOCK; + if self.initialized { + inline_block = self.inline_block; + } + + if versioned { + shard_size <= inline_block / 8 + } else { + shard_size <= inline_block + } } pub fn inline_block(&self) -> usize { @@ -161,15 +172,6 @@ impl Config { } } - pub fn inline_shard_limit_bytes(&self, versioned: bool) -> usize { - let inline_block = self.inline_block(); - if versioned { inline_block / 8 } else { inline_block } - } - - pub fn inline_object_limit_bytes(&self, data_shards: usize, versioned: bool) -> usize { - self.inline_shard_limit_bytes(versioned).saturating_mul(data_shards.max(1)) - } - pub fn capacity_optimized(&self) -> bool { if !self.initialized { false @@ -334,32 +336,3 @@ pub fn validate_parity_inner(ss_parity: usize, rrs_parity: usize, set_drive_coun } Ok(()) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn inline_object_limit_matches_default_non_versioned_budget() { - let cfg = Config { - initialized: true, - inline_block: DEFAULT_INLINE_BLOCK, - ..Default::default() - }; - - assert_eq!(cfg.inline_shard_limit_bytes(false), DEFAULT_INLINE_BLOCK); - assert_eq!(cfg.inline_object_limit_bytes(8, false), DEFAULT_INLINE_BLOCK * 8); - } - - #[test] - fn inline_object_limit_scales_down_for_versioned_objects() { - let cfg = Config { - initialized: true, - inline_block: DEFAULT_INLINE_BLOCK, - ..Default::default() - }; - - assert_eq!(cfg.inline_shard_limit_bytes(true), DEFAULT_INLINE_BLOCK / 8); - assert_eq!(cfg.inline_object_limit_bytes(8, true), DEFAULT_INLINE_BLOCK); - } -} diff --git a/crates/ecstore/src/data_movement.rs b/crates/ecstore/src/data_movement.rs index e3fa89b1e..f40840624 100644 --- a/crates/ecstore/src/data_movement.rs +++ b/crates/ecstore/src/data_movement.rs @@ -64,7 +64,7 @@ pub fn decode_part_index(index: Option<&Bytes>) -> Option { } } -pub fn put_data_from_chunk(chunk: Vec, size: i64, actual_size: i64, index: Option) -> Result { +pub fn put_obj_reader_from_chunk(chunk: Vec, size: i64, actual_size: i64, index: Option) -> Result { use sha2::{Digest, Sha256}; let sha256hex = if !chunk.is_empty() { @@ -74,7 +74,7 @@ pub fn put_data_from_chunk(chunk: Vec, size: i64, actual_size: i64, index: O }; let reader = IndexedDataMovementReader::new(Cursor::new(chunk), index); - let hash_reader = HashReader::from_reader(reader, size, actual_size, None, sha256hex, false)?; + let hash_reader = HashReader::from_stream(reader, size, actual_size, None, sha256hex, false)?; Ok(PutObjReader::new(hash_reader)) } @@ -172,7 +172,7 @@ pub(crate) async fn migrate_object( let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?; let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size }; let index = decode_part_index(part.index.as_ref()); - let mut data = put_data_from_chunk(chunk, part_size, part_actual_size, index)?; + let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?; let pi = match store .put_object_part( @@ -254,7 +254,7 @@ pub(crate) async fn migrate_object( .first() .and_then(|part| decode_part_index(part.index.as_ref())); let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index); - let hrd = HashReader::from_reader(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?; + let hrd = HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?; let mut data = PutObjReader::new(hrd); if let Err(err) = store diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 4548d3118..023a4c726 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -119,19 +119,18 @@ use tokio::{ time::{interval, timeout}, }; use tokio_util::sync::CancellationToken; +use tracing::error; +use tracing::{debug, info, warn}; +use uuid::Uuid; -const ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES: &str = "RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES"; -const ENV_RUSTFS_PUT_FORCE_DISABLE_INLINE: &str = "RUSTFS_PUT_FORCE_DISABLE_INLINE"; -const SLOW_PUT_STORAGE_PHASE_DEBUG_THRESHOLD_MS: u64 = 100; -const SLOW_PUT_STORAGE_PHASE_WARN_THRESHOLD_MS: u64 = 1_000; -const SLOW_PUT_STORAGE_PHASE_ERROR_THRESHOLD_MS: u64 = 5_000; +pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024; +pub const MAX_PARTS_COUNT: usize = 10000; +pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket"; +pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object"; -fn env_flag_enabled(name: &str) -> bool { - rustfs_utils::get_env_bool(name, false) -} - -fn env_non_negative_usize(name: &str) -> Option { - rustfs_utils::get_env_opt_usize(name) +pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap) { + metadata.remove(RUSTFS_MULTIPART_BUCKET_KEY); + metadata.remove(RUSTFS_MULTIPART_OBJECT_KEY); } fn capacity_scope_from_disks(disks: &[Option]) -> CapacityScope { @@ -164,65 +163,6 @@ fn record_capacity_scope_if_needed(scope_token: Option, disks: &[Option bool { - if !inline_by_topology || object_size < 0 { - return false; - } - - if env_flag_enabled(ENV_RUSTFS_PUT_FORCE_DISABLE_INLINE) { - return false; - } - - env_non_negative_usize(ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES) - .map(|value| usize::try_from(object_size).is_ok_and(|size| size <= value)) - .unwrap_or(inline_by_topology) -} - -fn log_put_storage_phase( - bucket: &str, - object: &str, - phase: &str, - elapsed: Duration, - object_size: i64, - inline_selected: bool, - write_quorum: usize, -) { - let duration_ms = elapsed.as_millis() as u64; - if duration_ms < SLOW_PUT_STORAGE_PHASE_DEBUG_THRESHOLD_MS { - return; - } - - if duration_ms >= SLOW_PUT_STORAGE_PHASE_ERROR_THRESHOLD_MS { - error!( - phase, - duration_ms, object_size, inline_selected, write_quorum, bucket, object, "PUT storage phase is critically slow" - ); - } else if duration_ms >= SLOW_PUT_STORAGE_PHASE_WARN_THRESHOLD_MS { - warn!( - phase, - duration_ms, object_size, inline_selected, write_quorum, bucket, object, "PUT storage phase is slow" - ); - } else { - debug!( - phase, - duration_ms, object_size, inline_selected, write_quorum, bucket, object, "PUT storage phase exceeded debug threshold" - ); - } -} -use tracing::error; -use tracing::{debug, info, warn}; -use uuid::Uuid; - -pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024; -pub const MAX_PARTS_COUNT: usize = 10000; -pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket"; -pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object"; - -pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap) { - metadata.remove(RUSTFS_MULTIPART_BUCKET_KEY); - metadata.remove(RUSTFS_MULTIPART_OBJECT_KEY); -} - /// Get the duplex buffer size from environment variable or use default. /// /// This function reads `RUSTFS_DUPLEX_BUFFER_SIZE` environment variable @@ -872,18 +812,13 @@ impl ObjectIO for SetDisks { let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); let is_inline_buffer = { - let inline_by_topology = if let Some(sc) = GLOBAL_STORAGE_CLASS.get() { + if let Some(sc) = GLOBAL_STORAGE_CLASS.get() { sc.should_inline(erasure.shard_file_size(data.size()), opts.versioned) } else { false - }; - resolved_put_inline_buffer_enabled(data.size(), inline_by_topology) + } }; - if is_inline_buffer { - rustfs_io_metrics::record_put_inline_selected(data.size(), opts.versioned); - } - let writer_setup_start = Instant::now(); let mut writers = Vec::with_capacity(shuffle_disks.len()); let mut errors = Vec::with_capacity(shuffle_disks.len()); for disk_op in shuffle_disks.iter() { @@ -917,15 +852,6 @@ impl ObjectIO for SetDisks { writers.push(None); } } - log_put_storage_phase( - bucket, - object, - "writer_setup", - writer_setup_start.elapsed(), - data.size(), - is_inline_buffer, - write_quorum, - ); let nil_count = errors.iter().filter(|&e| e.is_none()).count(); if nil_count < write_quorum { @@ -937,9 +863,6 @@ impl ObjectIO for SetDisks { return Err(Error::other(format!("not enough disks to write: {errors:?}"))); } - let object_size = data.size(); - let encode_write_start = Instant::now(); - let stream = mem::replace( &mut data.stream, HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?, @@ -948,30 +871,15 @@ impl ObjectIO for SetDisks { let (reader, w_size) = match Arc::new(erasure).encode(stream, &mut writers, write_quorum).await { Ok((r, w)) => (r, w), Err(e) => { - log_put_storage_phase( - bucket, - object, - "encode_write", - encode_write_start.elapsed(), - object_size, - is_inline_buffer, - write_quorum, - ); error!("encode err {:?}", e); return Err(e.into()); } }; // TODO: delete temporary directory on error let _ = mem::replace(&mut data.stream, reader); - log_put_storage_phase( - bucket, - object, - "encode_write", - encode_write_start.elapsed(), - object_size, - is_inline_buffer, - write_quorum, - ); + // if let Err(err) = close_bitrot_writers(&mut writers).await { + // error!("close_bitrot_writers err {:?}", err); + // } if (w_size as i64) < data.size() { warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size()); @@ -1047,7 +955,6 @@ impl ObjectIO for SetDisks { drop(writers); // drop writers to close all files, this is to prevent FileAccessDenied errors when renaming data - let post_write_lock_start = Instant::now(); if !opts.no_lock && object_lock_guard.is_none() { let ns_lock = self.new_ns_lock(bucket, object).await?; object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| { @@ -1057,17 +964,7 @@ impl ObjectIO for SetDisks { )) })?); } - log_put_storage_phase( - bucket, - object, - "post_write_lock", - post_write_lock_start.elapsed(), - data.size(), - is_inline_buffer, - write_quorum, - ); - let finalize_start = Instant::now(); let (online_disks, _, op_old_dir) = Self::rename_data( &shuffle_disks, RUSTFS_META_TMP_BUCKET, @@ -1077,18 +974,7 @@ impl ObjectIO for SetDisks { object, write_quorum, ) - .await - .inspect_err(|_| { - log_put_storage_phase( - bucket, - object, - "finalize", - finalize_start.elapsed(), - data.size(), - is_inline_buffer, - write_quorum, - ); - })?; + .await?; if let Some(old_dir) = op_old_dir { self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum) @@ -1098,15 +984,6 @@ impl ObjectIO for SetDisks { drop(object_lock_guard); // drop object lock guard to release the lock self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?; - log_put_storage_phase( - bucket, - object, - "finalize", - finalize_start.elapsed(), - data.size(), - is_inline_buffer, - write_quorum, - ); for (i, op_disk) in online_disks.iter().enumerate() { if let Some(disk) = op_disk @@ -2037,9 +1914,6 @@ impl ObjectOperations for SetDisks { if let Some(ref version_id) = opts.version_id { fi.version_id = Uuid::parse_str(version_id).ok(); } - if let Some(checksum) = &opts.resolved_checksum { - fi.checksum = Some(checksum.clone()); - } self.update_object_meta(bucket, object, fi.clone(), &online_disks) .await @@ -4403,31 +4277,6 @@ mod tests { } } - #[test] - #[serial] - fn resolved_put_inline_buffer_enabled_honors_disable_env() { - temp_env::with_var(ENV_RUSTFS_PUT_FORCE_DISABLE_INLINE, Some("true"), || { - assert!(!resolved_put_inline_buffer_enabled(4096, true)); - }); - } - - #[test] - #[serial] - fn resolved_put_inline_buffer_enabled_honors_max_bytes_override() { - temp_env::with_var(ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES, Some("4096"), || { - assert!(resolved_put_inline_buffer_enabled(4096, true)); - assert!(!resolved_put_inline_buffer_enabled(4097, true)); - }); - } - - #[test] - #[serial] - fn resolved_put_inline_buffer_enabled_ignores_invalid_override() { - temp_env::with_var(ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES, Some("invalid"), || { - assert!(resolved_put_inline_buffer_enabled(4096, true)); - }); - } - async fn current_setup_type() -> SetupType { if is_dist_erasure().await { SetupType::DistErasure diff --git a/crates/ecstore/src/store_api/types.rs b/crates/ecstore/src/store_api/types.rs index b569dca70..31446b84d 100644 --- a/crates/ecstore/src/store_api/types.rs +++ b/crates/ecstore/src/store_api/types.rs @@ -70,7 +70,6 @@ pub struct ObjectOptions { pub eval_metadata: Option>, - pub resolved_checksum: Option, pub want_checksum: Option, pub skip_verify_bitrot: bool, pub capacity_scope_token: Option, @@ -511,18 +510,6 @@ impl ObjectInfo { }) .collect(); - let actual_size = fi - .parts - .iter() - .map(|part| { - if part.actual_size > 0 { - part.actual_size - } else { - i64::try_from(part.size).unwrap_or_default() - } - }) - .sum(); - // TODO: part checksums ObjectInfo { @@ -535,7 +522,6 @@ impl ObjectInfo { delete_marker: fi.deleted, mod_time: fi.mod_time, size: fi.size, - actual_size, parts, is_latest: fi.is_latest, user_tags, diff --git a/crates/io-core/Cargo.toml b/crates/io-core/Cargo.toml index fd4fb14b5..81f932532 100644 --- a/crates/io-core/Cargo.toml +++ b/crates/io-core/Cargo.toml @@ -29,14 +29,12 @@ workspace = true [dependencies] bytes = { workspace = true } -futures-core = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync"] } memmap2 = { workspace = true } rustfs-io-metrics = { workspace = true } [dev-dependencies] -futures-util = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } [lib] diff --git a/crates/io-core/src/adapter.rs b/crates/io-core/src/adapter.rs deleted file mode 100644 index b2548382c..000000000 --- a/crates/io-core/src/adapter.rs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Compatibility adapter that exposes chunk streams as `AsyncRead`. - -use crate::chunk::BoxChunkStream; -use bytes::Bytes; -use std::io; -use std::pin::Pin; -use std::task::{Context, Poll}; -use tokio::io::{AsyncRead, ReadBuf}; - -/// `AsyncRead` adapter for boxed chunk streams. -pub struct ChunkStreamReader { - stream: BoxChunkStream, - current: Option, - offset: usize, -} - -impl ChunkStreamReader { - #[must_use] - pub fn new(stream: BoxChunkStream) -> Self { - Self { - stream, - current: None, - offset: 0, - } - } -} - -impl AsyncRead for ChunkStreamReader { - fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - loop { - if let Some(current) = &self.current { - if self.offset < current.len() { - let remaining = ¤t[self.offset..]; - let to_read = remaining.len().min(buf.remaining()); - buf.put_slice(&remaining[..to_read]); - self.offset += to_read; - return Poll::Ready(Ok(())); - } - - self.current = None; - self.offset = 0; - continue; - } - - match self.stream.as_mut().poll_next(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(Some(Ok(chunk))) => { - let next = chunk.as_bytes(); - if next.is_empty() { - continue; - } - self.current = Some(next); - self.offset = 0; - } - Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), - Poll::Ready(None) => return Poll::Ready(Ok(())), - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::chunk::{IoChunk, MappedChunk, PooledChunk}; - use bytes::Bytes; - use futures_util::stream; - use tokio::io::AsyncReadExt; - - #[tokio::test] - async fn test_chunk_stream_reader_reads_single_chunk() { - let stream: BoxChunkStream = Box::pin(stream::iter(vec![Ok(IoChunk::Shared(Bytes::from_static(b"hello")))])); - let mut reader = ChunkStreamReader::new(stream); - let mut out = Vec::new(); - reader.read_to_end(&mut out).await.unwrap(); - assert_eq!(out, b"hello"); - } - - #[tokio::test] - async fn test_chunk_stream_reader_reads_multiple_chunks() { - let stream: BoxChunkStream = Box::pin(stream::iter(vec![ - Ok(IoChunk::Shared(Bytes::from_static(b"he"))), - Ok(IoChunk::Mapped(MappedChunk::new(Bytes::from_static(b"llo!"), 0, 4).unwrap())), - Ok(IoChunk::Pooled(PooledChunk::from_bytes(Bytes::from_static(b" world")).unwrap())), - ])); - let mut reader = ChunkStreamReader::new(stream); - let mut out = Vec::new(); - reader.read_to_end(&mut out).await.unwrap(); - assert_eq!(out, b"hello! world"); - } - - #[tokio::test] - async fn test_chunk_stream_reader_handles_empty_stream() { - let stream: BoxChunkStream = Box::pin(stream::iter(Vec::>::new())); - let mut reader = ChunkStreamReader::new(stream); - let mut out = Vec::new(); - reader.read_to_end(&mut out).await.unwrap(); - assert!(out.is_empty()); - } - - #[tokio::test] - async fn test_chunk_stream_reader_propagates_stream_error() { - let stream: BoxChunkStream = Box::pin(stream::iter(vec![Err(io::Error::other("chunk stream failure"))])); - let mut reader = ChunkStreamReader::new(stream); - let mut out = Vec::new(); - let err = reader.read_to_end(&mut out).await.unwrap_err(); - assert_eq!(err.kind(), io::ErrorKind::Other); - assert!(err.to_string().contains("chunk stream failure")); - } -} diff --git a/crates/io-core/src/chunk.rs b/crates/io-core/src/chunk.rs deleted file mode 100644 index 4a3441cc5..000000000 --- a/crates/io-core/src/chunk.rs +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Core chunk ownership abstractions for the zero-copy data plane. - -use crate::pool::PooledBuffer; -use bytes::Bytes; -use futures_core::Stream; -use std::io; -use std::pin::Pin; - -/// Boxed asynchronous stream of I/O chunks. -pub type BoxChunkStream = Pin> + Send + Sync + 'static>>; - -/// Source of chunked data. -pub trait ChunkSource { - fn into_chunk_stream(self) -> BoxChunkStream - where - Self: Sized; -} - -/// Owned chunk variants used by the zero-copy data plane. -#[derive(Debug)] -pub enum IoChunk { - Shared(Bytes), - Mapped(MappedChunk), - Pooled(PooledChunk), -} - -impl IoChunk { - /// Returns the visible length of this chunk. - #[must_use] - pub fn len(&self) -> usize { - match self { - Self::Shared(bytes) => bytes.len(), - Self::Mapped(chunk) => chunk.len(), - Self::Pooled(chunk) => chunk.len(), - } - } - - /// Returns true when the chunk has no visible data. - #[must_use] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns a shared read-only view of the visible bytes. - #[must_use] - pub fn as_bytes(&self) -> Bytes { - match self { - Self::Shared(bytes) => bytes.clone(), - Self::Mapped(chunk) => chunk.as_bytes(), - Self::Pooled(chunk) => chunk.as_bytes(), - } - } - - /// Returns a sliced view relative to the currently visible bytes. - pub fn slice(&self, offset: usize, len: usize) -> io::Result { - match self { - Self::Shared(bytes) => { - validate_slice_bounds(bytes.len(), offset, len)?; - Ok(Self::Shared(bytes.slice(offset..offset + len))) - } - Self::Mapped(chunk) => chunk.slice(offset, len).map(Self::Mapped), - Self::Pooled(chunk) => chunk.slice(offset, len).map(Self::Pooled), - } - } -} - -/// Logical view into mapped file bytes. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MappedChunk { - bytes: Bytes, - logical_offset: usize, - logical_len: usize, -} - -impl MappedChunk { - pub fn new(bytes: Bytes, logical_offset: usize, logical_len: usize) -> io::Result { - validate_slice_bounds(bytes.len(), logical_offset, logical_len)?; - Ok(Self { - bytes, - logical_offset, - logical_len, - }) - } - - /// Returns the visible length of this mapped chunk. - #[must_use] - pub const fn len(&self) -> usize { - self.logical_len - } - - /// Returns true when the chunk has no visible data. - #[must_use] - pub const fn is_empty(&self) -> bool { - self.logical_len == 0 - } - - /// Returns the visible bytes for this logical view. - #[must_use] - pub fn as_bytes(&self) -> Bytes { - self.bytes - .slice(self.logical_offset..self.logical_offset.saturating_add(self.logical_len)) - } - - /// Returns a sliced logical view relative to the current logical view. - pub fn slice(&self, offset: usize, len: usize) -> io::Result { - validate_slice_bounds(self.logical_len, offset, len)?; - Self::new(self.bytes.clone(), self.logical_offset + offset, len) - } -} - -/// Placeholder pooled chunk variant for commit 4. -/// -/// This is backed by a `PooledBuffer` and exposes a visible read-only window. -#[derive(Debug)] -pub struct PooledChunk { - bytes: Bytes, -} - -#[derive(Debug)] -struct PooledChunkOwner { - buffer: PooledBuffer, - visible_len: usize, -} - -impl AsRef<[u8]> for PooledChunkOwner { - fn as_ref(&self) -> &[u8] { - &self.buffer[..self.visible_len] - } -} - -#[derive(Debug)] -struct DetachedVecChunkOwner { - bytes: Vec, -} - -impl AsRef<[u8]> for DetachedVecChunkOwner { - fn as_ref(&self) -> &[u8] { - &self.bytes - } -} - -impl PooledChunk { - pub fn new(buffer: PooledBuffer, len: usize) -> io::Result { - validate_slice_bounds(buffer.len(), 0, len)?; - Ok(Self { - bytes: Bytes::from_owner(PooledChunkOwner { - buffer, - visible_len: len, - }), - }) - } - - /// Convenience constructor for detached test and compatibility values. - pub fn from_bytes(bytes: Bytes) -> io::Result { - let len = bytes.len(); - Self::new(PooledBuffer::from_bytes(bytes), len) - } - - /// Detached constructor that takes ownership of an existing `Vec` - /// without introducing an additional copy. - pub fn from_vec(bytes: Vec) -> Self { - Self { - bytes: Bytes::from_owner(DetachedVecChunkOwner { bytes }), - } - } - - /// Returns the visible length of this pooled chunk. - #[must_use] - pub fn len(&self) -> usize { - self.bytes.len() - } - - /// Returns true when the chunk has no visible data. - #[must_use] - pub fn is_empty(&self) -> bool { - self.bytes.is_empty() - } - - /// Returns the visible bytes for this pooled chunk. - #[must_use] - pub fn as_bytes(&self) -> Bytes { - self.bytes.clone() - } - - /// Returns a sliced pooled chunk relative to the current visible view. - pub fn slice(&self, offset: usize, len: usize) -> io::Result { - validate_slice_bounds(self.bytes.len(), offset, len)?; - Ok(Self { - bytes: self.bytes.slice(offset..offset + len), - }) - } -} - -fn validate_slice_bounds(visible_len: usize, offset: usize, len: usize) -> io::Result<()> { - let end = offset - .checked_add(len) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "chunk slice overflows"))?; - if end > visible_len { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "chunk slice exceeds visible length")); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pool::BytesPool; - - #[test] - fn test_shared_chunk_len_and_slice() { - let chunk = IoChunk::Shared(Bytes::from_static(b"abcdef")); - assert_eq!(chunk.len(), 6); - assert!(!chunk.is_empty()); - assert_eq!(chunk.as_bytes(), Bytes::from_static(b"abcdef")); - assert_eq!(chunk.slice(1, 3).unwrap().as_bytes(), Bytes::from_static(b"bcd")); - } - - #[test] - fn test_mapped_chunk_len_and_slice() { - let chunk = MappedChunk::new(Bytes::from_static(b"abcdefgh"), 2, 4).unwrap(); - assert_eq!(chunk.len(), 4); - assert_eq!(chunk.as_bytes(), Bytes::from_static(b"cdef")); - assert_eq!(chunk.slice(1, 2).unwrap().as_bytes(), Bytes::from_static(b"de")); - } - - #[test] - fn test_pooled_chunk_len_and_as_bytes() { - let chunk = PooledChunk::from_bytes(Bytes::from_static(b"hello")).unwrap(); - assert_eq!(chunk.len(), 5); - assert_eq!(chunk.as_bytes(), Bytes::from_static(b"hello")); - assert_eq!(chunk.slice(1, 3).unwrap().as_bytes(), Bytes::from_static(b"ell")); - } - - #[test] - fn test_io_chunk_as_bytes_for_all_variants() { - let shared = IoChunk::Shared(Bytes::from_static(b"s")); - let mapped = IoChunk::Mapped(MappedChunk::new(Bytes::from_static(b"mapped"), 0, 6).unwrap()); - let pooled = IoChunk::Pooled(PooledChunk::from_bytes(Bytes::from_static(b"p")).unwrap()); - - assert_eq!(shared.as_bytes(), Bytes::from_static(b"s")); - assert_eq!(mapped.as_bytes(), Bytes::from_static(b"mapped")); - assert_eq!(pooled.as_bytes(), Bytes::from_static(b"p")); - } - - #[tokio::test] - async fn test_pooled_chunk_keeps_owner_alive_until_last_view_drops() { - let pool = BytesPool::new_tiered(); - let mut buffer = pool.acquire_buffer(16).await; - buffer.extend_from_slice(b"pooled-bytes"); - - let chunk = PooledChunk::new(buffer, "pooled-bytes".len()).unwrap(); - let bytes = chunk.as_bytes(); - - assert_eq!(pool.available_buffers(), 0); - drop(chunk); - assert_eq!(pool.available_buffers(), 0); - assert_eq!(bytes, Bytes::from_static(b"pooled-bytes")); - - drop(bytes); - assert_eq!(pool.available_buffers(), 1); - } -} diff --git a/crates/io-core/src/lib.rs b/crates/io-core/src/lib.rs index f8565d50e..fe0ee031f 100644 --- a/crates/io-core/src/lib.rs +++ b/crates/io-core/src/lib.rs @@ -46,10 +46,8 @@ //! let mut buffer = pool.acquire_buffer(8192).await; //! ``` -pub mod adapter; pub mod backpressure; pub mod bufreader_optimizer; -pub mod chunk; pub mod config; pub mod deadlock_detector; pub mod direct_io; @@ -70,9 +68,7 @@ pub use reader::{ZeroCopyObjectReader, ZeroCopyReadError}; pub use writer::{ZeroCopyObjectWriter, ZeroCopyWriteError}; // BufReader optimizer exports -pub use adapter::ChunkStreamReader; pub use bufreader_optimizer::{BufReaderConfig, BufReaderOptimizer, BufReaderStats, BufferedSource}; -pub use chunk::{BoxChunkStream, ChunkSource, IoChunk, MappedChunk, PooledChunk}; // Shared memory exports pub use shared_memory::{ArcData, ArcMetadata, SharedMemoryConfig, SharedMemoryPool, SharedMemoryStats}; diff --git a/crates/io-core/src/pool.rs b/crates/io-core/src/pool.rs index fc5e642f3..aa08fcfde 100644 --- a/crates/io-core/src/pool.rs +++ b/crates/io-core/src/pool.rs @@ -17,7 +17,7 @@ //! Migrated from rustfs-ecstore to provide unified buffer pooling //! across rustfs and rustfs-ecstore without cyclic dependencies. -use bytes::{Bytes, BytesMut}; +use bytes::BytesMut; use std::mem::ManuallyDrop; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -108,7 +108,6 @@ pub struct BytesPoolMetrics { /// A buffer managed by the BytesPool. /// /// When dropped, the buffer is automatically returned to the pool for reuse. -#[derive(Debug)] pub struct PooledBuffer { /// The underlying buffer (ManuallyDrop to allow taking on drop) pub buffer: ManuallyDrop, @@ -118,45 +117,6 @@ pub struct PooledBuffer { _permit: Option, } -impl PooledBuffer { - /// Create a detached pooled buffer from bytes. - /// - /// This is primarily used for tests and transitional adapters where the - /// chunk abstraction needs a pool-shaped owner before a real pool-backed - /// producer exists. - #[must_use] - pub fn from_bytes(bytes: Bytes) -> Self { - Self { - buffer: ManuallyDrop::new(BytesMut::from(bytes.as_ref())), - tier: None, - _permit: None, - } - } - - /// Current visible length of the underlying buffer. - #[must_use] - pub fn len(&self) -> usize { - self.buffer.len() - } - - /// Total buffer capacity. - #[must_use] - pub fn capacity(&self) -> usize { - self.buffer.capacity() - } - - /// Clear the visible contents while preserving capacity. - pub fn clear(&mut self) { - self.buffer.clear(); - } - - /// Returns true when the visible buffer is empty. - #[must_use] - pub fn is_empty(&self) -> bool { - self.buffer.is_empty() - } -} - /// BytesPool configuration. /// /// Allows customization of buffer sizes and limits for each tier. diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index bdfa0020d..c61fca193 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -121,129 +121,8 @@ pub use config::{ // Re-exports for convenience pub use collector::MetricsCollector; -pub use metric_names::data_plane; pub use performance::PerformanceMetrics; -/// High-level request path selected for an I/O operation. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum IoPath { - Fast, - Legacy, -} - -impl IoPath { - #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::Fast => "fast", - Self::Legacy => "legacy", - } - } -} - -/// Effective copy mode observed for an I/O operation. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CopyMode { - TrueZeroCopy, - SharedBytes, - SingleCopy, - Reconstructed, - Transformed, -} - -impl CopyMode { - #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::TrueZeroCopy => "true_zero_copy", - Self::SharedBytes => "shared_bytes", - Self::SingleCopy => "single_copy", - Self::Reconstructed => "reconstructed", - Self::Transformed => "transformed", - } - } -} - -/// Stage where a data plane decision or fallback happened. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum IoStage { - Unknown, - ReadSetup, - HttpBridge, - LocalDiskChunk, - RangeGuard, - PutTransform, -} - -impl IoStage { - #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::Unknown => "unknown", - Self::ReadSetup => "read_setup", - Self::HttpBridge => "http_bridge", - Self::LocalDiskChunk => "local_disk_chunk", - Self::RangeGuard => "range_guard", - Self::PutTransform => "put_transform", - } - } -} - -/// Reason why the data plane fell back from a preferred path. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FallbackReason { - Unknown, - ProbeFailed, - MmapDisabled, - MmapUnavailable, - SmallObject, - WindowLimitExceeded, - UnalignedWindow, - RangeNotSupported, - EncryptionEnabled, - CompressionEnabled, - TransformEncryptionLegacy, - TransformCompressionLegacy, - TransformCompressionEncryptionLegacy, - ChunkBridgeUnavailable, - NonLocalBackend, -} - -impl FallbackReason { - #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::Unknown => "unknown", - Self::ProbeFailed => "probe_failed", - Self::MmapDisabled => "mmap_disabled", - Self::MmapUnavailable => "mmap_unavailable", - Self::SmallObject => "small_object", - Self::WindowLimitExceeded => "window_limit_exceeded", - Self::UnalignedWindow => "unaligned_window", - Self::RangeNotSupported => "range_not_supported", - Self::EncryptionEnabled => "encryption_enabled", - Self::CompressionEnabled => "compression_enabled", - Self::TransformEncryptionLegacy => "transform_encryption_legacy", - Self::TransformCompressionLegacy => "transform_compression_legacy", - Self::TransformCompressionEncryptionLegacy => "transform_compression_encryption_legacy", - Self::ChunkBridgeUnavailable => "chunk_bridge_unavailable", - Self::NonLocalBackend => "non_local_backend", - } - } -} - -#[inline(always)] -fn put_size_bucket_label(size_bytes: i64) -> &'static str { - match size_bytes { - ..=0 => "unknown", - 1..=16_384 => "le_16kib", - 16_385..=65_536 => "le_64kib", - 65_537..=262_144 => "le_256kib", - 262_145..=1_048_576 => "le_1mib", - _ => "gt_1mib", - } -} - /// Record GetObject request start. #[inline(always)] pub fn record_get_object_request_start(concurrent_requests: usize) { @@ -316,138 +195,40 @@ pub fn record_get_object_io_state( counter!("rustfs_io_strategy_selected_total", "level" => load_level.to_string()).increment(1); } -/// Record which request path was selected for an operation. +/// Record a zero-copy read operation. +/// +/// # Arguments +/// +/// * `size_bytes` - Size of the data read in bytes +/// * `duration_ms` - Time taken for the read operation in milliseconds #[inline(always)] -pub fn record_io_path_selected(operation: &'static str, io_path: IoPath) { - counter!( - metric_names::data_plane::PATH_SELECTED_TOTAL, - "path" => operation, - "mode" => io_path.as_str() - ) - .increment(1); +pub fn record_zero_copy_read(size_bytes: usize, duration_ms: f64) { + counter!("rustfs.zero_copy.reads.total").increment(1); + histogram!("rustfs.zero_copy.read.size.bytes").record(size_bytes as f64); + histogram!("rustfs.zero_copy.read.duration.ms").record(duration_ms); } -/// Record the effective copy mode for an operation. +/// Record memory copies avoided by using zero-copy. +/// +/// # Arguments +/// +/// * `bytes_saved` - Number of bytes that would have been copied without zero-copy #[inline(always)] -pub fn record_io_copy_mode(operation: &'static str, copy_mode: CopyMode, size_bytes: usize) { - counter!( - metric_names::data_plane::COPY_MODE_BYTES_TOTAL, - "path" => operation, - "mode" => copy_mode.as_str() - ) - .increment(size_bytes as u64); +pub fn record_memory_copy_saved(bytes_saved: usize) { + counter!("rustfs.zero_copy.memory.saved.bytes").increment(bytes_saved as u64); } -/// Record a data plane fallback decision. +/// Record a fallback from zero-copy to regular read. +/// +/// This happens when zero-copy read fails (e.g., mmap not available, +/// file too large, etc.) and the system falls back to regular I/O. +/// +/// # Arguments +/// +/// * `reason` - Reason for the fallback (e.g., "mmap_unavailable", "file_too_large") #[inline(always)] -pub fn record_io_fallback(stage: IoStage, reason: FallbackReason) { - counter!( - metric_names::data_plane::FALLBACK_TOTAL, - "stage" => stage.as_str(), - "reason" => reason.as_str() - ) - .increment(1); -} - -/// Record the currently active mmap bytes held by LocalDisk chunk streams. -#[inline(always)] -pub fn record_local_disk_active_mmap_bytes(active_bytes: usize) { - gauge!(metric_names::data_plane::LOCAL_DISK_ACTIVE_MMAP_BYTES).set(active_bytes as f64); -} - -/// Record pooled chunk usage in LocalDisk compatibility paths. -#[inline(always)] -pub fn record_local_disk_pooled_chunk(source: &'static str, size_bytes: usize) { - counter!( - metric_names::data_plane::LOCAL_DISK_POOLED_CHUNKS_TOTAL, - "source" => source - ) - .increment(1); - counter!( - metric_names::data_plane::LOCAL_DISK_POOLED_BYTES_TOTAL, - "source" => source - ) - .increment(size_bytes as u64); -} - -/// Record a compatibility chunk-stream aggregation performed by `read_file_zero_copy()`. -#[inline(always)] -pub fn record_local_disk_compat_collect(chunk_count: usize, total_bytes: usize) { - counter!(metric_names::data_plane::LOCAL_DISK_COMPAT_COLLECT_TOTAL).increment(1); - histogram!(metric_names::data_plane::LOCAL_DISK_COMPAT_COLLECT_CHUNKS).record(chunk_count as f64); - histogram!(metric_names::data_plane::LOCAL_DISK_COMPAT_COLLECT_BYTES).record(total_bytes as f64); -} - -/// Record an attempted PUT fast path. -#[inline(always)] -pub fn record_put_object_attempted_fast_path(size_bytes: i64) { - counter!(metric_names::data_plane::PUT_FAST_PATH_ATTEMPTS_TOTAL).increment(1); - - if size_bytes > 0 { - histogram!(metric_names::data_plane::PUT_FAST_PATH_ATTEMPT_SIZE_BYTES).record(size_bytes as f64); - } -} - -/// Record which transformed PUT pipeline was selected. -#[inline(always)] -pub fn record_put_transform_selected(kind: &'static str, io_path: IoPath, size_bytes: usize) { - counter!( - metric_names::data_plane::PUT_TRANSFORM_SELECTED_TOTAL, - "kind" => kind, - "mode" => io_path.as_str() - ) - .increment(1); - - histogram!( - metric_names::data_plane::PUT_TRANSFORM_SIZE_BYTES, - "kind" => kind, - "mode" => io_path.as_str() - ) - .record(size_bytes as f64); -} - -/// Record PUT path selection with size-bucket context. -#[inline(always)] -pub fn record_put_path_selected(size_bytes: i64, io_path: IoPath) { - counter!( - "rustfs.s3.put_object.path.selected.total", - "mode" => io_path.as_str(), - "size_bucket" => put_size_bucket_label(size_bytes) - ) - .increment(1); -} - -/// Record PUT copy mode with size-bucket context. -#[inline(always)] -pub fn record_put_copy_mode(size_bytes: i64, copy_mode: CopyMode) { - counter!( - "rustfs.s3.put_object.copy_mode.total", - "mode" => copy_mode.as_str(), - "size_bucket" => put_size_bucket_label(size_bytes) - ) - .increment(1); -} - -/// Record PUT fallback with size-bucket context. -#[inline(always)] -pub fn record_put_fallback(size_bytes: i64, reason: FallbackReason) { - counter!( - "rustfs.s3.put_object.fallback.total", - "reason" => reason.as_str(), - "size_bucket" => put_size_bucket_label(size_bytes) - ) - .increment(1); -} - -/// Record inline-object selection for PUT with size-bucket context. -#[inline(always)] -pub fn record_put_inline_selected(size_bytes: i64, versioned: bool) { - counter!( - "rustfs.s3.put_object.inline.selected.total", - "versioned" => if versioned { "true" } else { "false" }, - "size_bucket" => put_size_bucket_label(size_bytes) - ) - .increment(1); +pub fn record_zero_copy_fallback(reason: &str) { + counter!("rustfs.zero_copy.fallback.total", "reason" => reason.to_string()).increment(1); } // ============================================================================ @@ -505,6 +286,41 @@ pub fn record_bytes_pool_hit_rate(tier: &str, hit_rate: f64) { gauge!("rustfs.bytes.pool.hit.rate", "tier" => tier.to_string()).set(hit_rate * 100.0); } +/// Record zero-copy write operation. +/// +/// # Arguments +/// +/// * `size_bytes` - Size of the data written in bytes +/// * `duration_ms` - Time taken for the write operation in milliseconds +#[inline(always)] +pub fn record_zero_copy_write(size_bytes: usize, duration_ms: f64) { + counter!("rustfs.zero_copy.write.total").increment(1); + histogram!("rustfs.zero_copy.write.size.bytes").record(size_bytes as f64); + histogram!("rustfs.zero_copy.write.duration.ms").record(duration_ms); +} + +/// Record zero-copy write fallback. +/// +/// This happens when zero-copy write fails and the system falls back to regular I/O. +/// +/// # Arguments +/// +/// * `reason` - Reason for the fallback +#[inline(always)] +pub fn record_zero_copy_write_fallback(reason: &str) { + counter!("rustfs.zero_copy.write.fallback.total", "reason" => reason.to_string()).increment(1); +} + +/// Record bytes saved from zero-copy. +/// +/// # Arguments +/// +/// * `size_bytes` - Number of bytes saved from zero-copy +#[inline(always)] +pub fn record_bytes_saved(size_bytes: usize) { + counter!("rustfs.zero_copy.bytes.saved.total").increment(size_bytes as u64); +} + // ============================================================================ // S3 Operation Metrics (GetObject, PutObject, etc.) // ============================================================================ @@ -534,33 +350,14 @@ pub fn record_get_object(duration_ms: f64, size_bytes: i64) { /// /// * `duration_ms` - Operation duration in milliseconds /// * `size_bytes` - Object size in bytes -/// * `zero_copy_enabled` - Legacy aggregate flag preserved for compatibility -/// -/// Note: this function records aggregate S3 PUT metrics only. The definitive -/// outcome of request-level fast-path attempts must be tracked separately via -/// ADR 0001 data-plane helpers. +/// * `zero_copy_enabled` - Whether zero-copy was enabled for this operation #[inline(always)] pub fn record_put_object(duration_ms: f64, size_bytes: i64, zero_copy_enabled: bool) { counter!("rustfs.s3.put_object.total").increment(1); histogram!("rustfs.s3.put_object.duration.ms").record(duration_ms); - counter!( - "rustfs.s3.put_object.bucketed.total", - "size_bucket" => put_size_bucket_label(size_bytes) - ) - .increment(1); - histogram!( - "rustfs.s3.put_object.bucketed.duration.ms", - "size_bucket" => put_size_bucket_label(size_bytes) - ) - .record(duration_ms); if size_bytes > 0 { histogram!("rustfs.s3.put_object.size.bytes").record(size_bytes as f64); - histogram!( - "rustfs.s3.put_object.bucketed.size.bytes", - "size_bucket" => put_size_bucket_label(size_bytes) - ) - .record(size_bytes as f64); } if zero_copy_enabled { @@ -855,146 +652,15 @@ pub fn record_io_latency_p99(latency_ms: f64) { gauge!("rustfs.io.latency.p99.ms").set(latency_ms); } -// ============================================================================ -// Zero-Copy I/O Metrics -// ============================================================================ - -/// Record a successful zero-copy read operation (e.g., mmap). -/// -/// # Arguments -/// -/// * `size_bytes` - Size of the data read in bytes -/// * `duration_ms` - Time taken for the read operation in milliseconds -#[inline(always)] -pub fn record_zero_copy_read(size_bytes: usize, duration_ms: f64) { - counter!("rustfs.zero_copy.reads.total").increment(1); - histogram!("rustfs.zero_copy.read.size.bytes").record(size_bytes as f64); - histogram!("rustfs.zero_copy.read.duration.ms").record(duration_ms); -} - -/// Record a fallback from zero-copy to regular read. -/// -/// This happens when zero-copy read fails (e.g., mmap not available, -/// file too large, etc.) and the system falls back to regular I/O. -/// -/// # Arguments -/// -/// * `reason` - Reason for the fallback (e.g., "mmap_unavailable", "file_too_large") -#[inline(always)] -pub fn record_zero_copy_fallback(reason: &str) { - counter!("rustfs.zero_copy.fallback.total", "reason" => reason.to_string()).increment(1); -} - #[cfg(test)] mod tests { use super::*; #[test] - fn test_io_path_as_str_values_stable() { - assert_eq!(IoPath::Fast.as_str(), "fast"); - assert_eq!(IoPath::Legacy.as_str(), "legacy"); - } - - #[test] - fn test_copy_mode_as_str_values_stable() { - assert_eq!(CopyMode::TrueZeroCopy.as_str(), "true_zero_copy"); - assert_eq!(CopyMode::SharedBytes.as_str(), "shared_bytes"); - assert_eq!(CopyMode::SingleCopy.as_str(), "single_copy"); - assert_eq!(CopyMode::Reconstructed.as_str(), "reconstructed"); - assert_eq!(CopyMode::Transformed.as_str(), "transformed"); - } - - #[test] - fn test_fallback_reason_as_str_values_stable() { - assert_eq!(FallbackReason::Unknown.as_str(), "unknown"); - assert_eq!(FallbackReason::ProbeFailed.as_str(), "probe_failed"); - assert_eq!(FallbackReason::MmapDisabled.as_str(), "mmap_disabled"); - assert_eq!(FallbackReason::MmapUnavailable.as_str(), "mmap_unavailable"); - assert_eq!(FallbackReason::SmallObject.as_str(), "small_object"); - assert_eq!(FallbackReason::WindowLimitExceeded.as_str(), "window_limit_exceeded"); - assert_eq!(FallbackReason::UnalignedWindow.as_str(), "unaligned_window"); - assert_eq!(FallbackReason::RangeNotSupported.as_str(), "range_not_supported"); - assert_eq!(FallbackReason::EncryptionEnabled.as_str(), "encryption_enabled"); - assert_eq!(FallbackReason::CompressionEnabled.as_str(), "compression_enabled"); - assert_eq!(FallbackReason::TransformEncryptionLegacy.as_str(), "transform_encryption_legacy"); - assert_eq!(FallbackReason::TransformCompressionLegacy.as_str(), "transform_compression_legacy"); - assert_eq!( - FallbackReason::TransformCompressionEncryptionLegacy.as_str(), - "transform_compression_encryption_legacy" - ); - assert_eq!(FallbackReason::ChunkBridgeUnavailable.as_str(), "chunk_bridge_unavailable"); - assert_eq!(FallbackReason::NonLocalBackend.as_str(), "non_local_backend"); - } - - #[test] - fn test_record_io_path_selected() { - record_io_path_selected("get", IoPath::Fast); - record_io_path_selected("put", IoPath::Legacy); - } - - #[test] - fn test_record_io_copy_mode() { - record_io_copy_mode("get", CopyMode::SharedBytes, 1024); - record_io_copy_mode("put", CopyMode::Transformed, 2048); - } - - #[test] - fn test_record_io_fallback() { - record_io_fallback(IoStage::ReadSetup, FallbackReason::MmapUnavailable); - record_io_fallback(IoStage::HttpBridge, FallbackReason::ChunkBridgeUnavailable); - } - - #[test] - fn test_record_local_disk_active_mmap_bytes() { - record_local_disk_active_mmap_bytes(4096); - record_local_disk_active_mmap_bytes(0); - } - - #[test] - fn test_record_local_disk_pooled_chunk() { - record_local_disk_pooled_chunk("fallback", 4096); - record_local_disk_pooled_chunk("compat_collect", 8192); - } - - #[test] - fn test_record_local_disk_compat_collect() { - record_local_disk_compat_collect(3, 16384); - } - - #[test] - fn test_record_put_object_attempted_fast_path() { - record_put_object_attempted_fast_path(1024 * 1024); - record_put_object_attempted_fast_path(0); - } - - #[test] - fn test_record_put_transform_selected() { - record_put_transform_selected("compression", IoPath::Fast, 2048); - record_put_transform_selected("compression_encryption", IoPath::Legacy, 4096); - } - - #[test] - fn test_record_put_path_selected() { - record_put_path_selected(8 * 1024, IoPath::Fast); - record_put_path_selected(2 * 1024 * 1024, IoPath::Legacy); - } - - #[test] - fn test_record_put_copy_mode() { - record_put_copy_mode(8 * 1024, CopyMode::SingleCopy); - record_put_copy_mode(512 * 1024, CopyMode::Transformed); - } - - #[test] - fn test_record_put_fallback() { - record_put_fallback(32 * 1024, FallbackReason::CompressionEnabled); - record_put_fallback(2 * 1024 * 1024, FallbackReason::EncryptionEnabled); - } - - #[test] - fn test_record_put_inline_selected() { - record_put_inline_selected(8 * 1024, false); - record_put_inline_selected(32 * 1024, true); + fn test_record_zero_copy_read() { + record_zero_copy_read(1024, 10.5); + record_memory_copy_saved(1024); + record_zero_copy_fallback("test"); } #[test] @@ -1005,6 +671,13 @@ mod tests { record_bytes_pool_hit_rate("small", 0.85); } + #[test] + fn test_record_zero_copy_write() { + record_zero_copy_write(1024, 10.5); + record_zero_copy_write_fallback("test"); + record_bytes_saved(1024); + } + // S3 Operation Metrics Tests #[test] fn test_record_get_object() { @@ -1109,6 +782,157 @@ mod tests { } } +// ============================================================================ +// Zero-Copy Optimization Metrics (Phase 1 Extension) +// ============================================================================ + pub mod bandwidth; pub mod global_metrics; pub mod metric_names; + +pub use metric_names::zero_copy; + +/// Record a zero-copy buffer operation. +/// +/// This function records metrics for zero-copy buffer operations, +/// including the operation type and size. +#[inline(always)] +pub fn record_zero_copy_buffer_operation(operation: &str, size: usize) { + counter!( + zero_copy::BUFFER_OPERATIONS_TOTAL, + "operation" => operation.to_string() + ) + .increment(1); + + counter!( + zero_copy::BUFFER_BYTES_TOTAL, + "operation" => operation.to_string() + ) + .increment(size as u64); +} + +/// Record memory copy operations. +/// +/// This function tracks the number and size of memory copies, +/// which should be minimized in zero-copy paths. +#[inline(always)] +pub fn record_memory_copy(count: u32, size: usize) { + counter!(zero_copy::MEMORY_COPY_TOTAL).increment(count as u64); + + counter!(zero_copy::MEMORY_COPY_BYTES_TOTAL).increment(size as u64); + + histogram!("rustfs_memory_copy_size_bytes").record(size as f64); +} + +/// Record a shared reference operation. +/// +/// This function tracks operations that create or use shared references +/// for zero-copy data sharing. +#[inline(always)] +pub fn record_shared_ref_operation(operation: &str) { + counter!( + zero_copy::SHARED_REF_OPERATIONS_TOTAL, + "operation" => operation.to_string() + ) + .increment(1); +} + +/// Record BufReader optimization. +/// +/// This function tracks BufReader layer elimination and buffer size +/// adjustments. +#[inline(always)] +pub fn record_bufreader_optimization(layers_eliminated: u32, buffer_size: usize) { + counter!(zero_copy::BUFREADER_LAYERS_ELIMINATED_TOTAL).increment(layers_eliminated as u64); + + histogram!(zero_copy::BUFREADER_BUFFER_SIZE_BYTES).record(buffer_size as f64); +} + +/// Record Direct I/O operation. +/// +/// This function tracks Direct I/O operations and their success/fallback +/// status. +#[inline(always)] +pub fn record_direct_io_operation(operation: &str, size: usize, success: bool) { + let status = if success { "success" } else { "fallback" }; + + counter!( + zero_copy::DIRECT_IO_OPERATIONS_TOTAL, + "operation" => operation.to_string(), + "status" => status.to_string() + ) + .increment(1); + + counter!( + zero_copy::DIRECT_IO_BYTES_TOTAL, + "operation" => operation.to_string(), + "status" => status.to_string() + ) + .increment(size as u64); +} + +/// Update zero-copy performance metrics. +/// +/// This function updates gauge metrics for overall zero-copy performance. +#[inline(always)] +pub fn update_zero_copy_performance_metrics(copy_count: u32, throughput_mbps: f64, memory_saved: u64) { + gauge!(zero_copy::AVG_COPY_COUNT).set(copy_count as f64); + + gauge!(zero_copy::THROUGHPUT_MBPS).set(throughput_mbps); + + gauge!(zero_copy::MEMORY_SAVED_BYTES).set(memory_saved as f64); +} + +// ============================================================================ +// Zero-Copy Metrics Tests +// ============================================================================ + +#[cfg(test)] +mod zero_copy_tests { + use super::*; + + #[test] + fn test_record_zero_copy_buffer_operation() { + // This test verifies the function compiles and runs + // Actual metric verification requires a metrics recorder + record_zero_copy_buffer_operation("read", 1024); + record_zero_copy_buffer_operation("write", 2048); + } + + #[test] + fn test_record_memory_copy() { + record_memory_copy(1, 1024); + record_memory_copy(2, 2048); + } + + #[test] + fn test_record_shared_ref_operation() { + record_shared_ref_operation("create"); + record_shared_ref_operation("share"); + } + + #[test] + fn test_record_bufreader_optimization() { + record_bufreader_optimization(1, 8192); + record_bufreader_optimization(2, 65536); + } + + #[test] + fn test_record_direct_io_operation() { + record_direct_io_operation("read", 4096, true); + record_direct_io_operation("write", 8192, false); + } + + #[test] + fn test_update_zero_copy_performance_metrics() { + update_zero_copy_performance_metrics(2, 150.5, 1024 * 1024); + } + + #[test] + fn test_metric_names() { + // Verify metric names are defined + assert!(!zero_copy::BUFFER_OPERATIONS_TOTAL.is_empty()); + assert!(!zero_copy::MEMORY_COPY_TOTAL.is_empty()); + assert!(!zero_copy::THROUGHPUT_MBPS.is_empty()); + } +} diff --git a/crates/io-metrics/src/metric_names.rs b/crates/io-metrics/src/metric_names.rs index 6c611b6aa..e7581ff8f 100644 --- a/crates/io-metrics/src/metric_names.rs +++ b/crates/io-metrics/src/metric_names.rs @@ -14,44 +14,41 @@ //! Metric name constants for consistent naming across the codebase. -/// Request-level data plane metric names introduced by ADR 0001. -pub mod data_plane { - /// Total number of selected request paths. - pub const PATH_SELECTED_TOTAL: &str = "rustfs.io.path.selected_total"; +/// Zero-copy operation metric names. +pub mod zero_copy { + /// Total number of zero-copy buffer operations + pub const BUFFER_OPERATIONS_TOTAL: &str = "rustfs_zero_copy_buffer_operations_total"; - /// Total bytes observed for a given effective copy mode. - pub const COPY_MODE_BYTES_TOTAL: &str = "rustfs.io.copy_mode.bytes_total"; + /// Total bytes processed by zero-copy buffer operations + pub const BUFFER_BYTES_TOTAL: &str = "rustfs_zero_copy_buffer_bytes_total"; - /// Total number of data plane fallbacks. - pub const FALLBACK_TOTAL: &str = "rustfs.io.zero_copy.fallback_total"; + /// Total number of memory copies + pub const MEMORY_COPY_TOTAL: &str = "rustfs_memory_copy_total"; - /// Current active local-disk mmap bytes held by chunk fast paths. - pub const LOCAL_DISK_ACTIVE_MMAP_BYTES: &str = "rustfs.io.local_disk.active_mmap.bytes"; + /// Total bytes copied in memory + pub const MEMORY_COPY_BYTES_TOTAL: &str = "rustfs_memory_copy_bytes_total"; - /// Total pooled chunks produced or consumed by LocalDisk compatibility paths. - pub const LOCAL_DISK_POOLED_CHUNKS_TOTAL: &str = "rustfs.io.local_disk.pooled_chunks.total"; + /// Total number of shared reference operations + pub const SHARED_REF_OPERATIONS_TOTAL: &str = "rustfs_shared_ref_operations_total"; - /// Total pooled bytes produced or consumed by LocalDisk compatibility paths. - pub const LOCAL_DISK_POOLED_BYTES_TOTAL: &str = "rustfs.io.local_disk.pooled_bytes.total"; + /// Total number of BufReader layers eliminated + pub const BUFREADER_LAYERS_ELIMINATED_TOTAL: &str = "rustfs_bufreader_layers_eliminated_total"; - /// Total number of compatibility chunk-stream aggregations performed for LocalDisk reads. - pub const LOCAL_DISK_COMPAT_COLLECT_TOTAL: &str = "rustfs.io.local_disk.compat_collect.total"; + /// BufReader buffer size distribution + pub const BUFREADER_BUFFER_SIZE_BYTES: &str = "rustfs_bufreader_buffer_size_bytes"; - /// Chunk count distribution for LocalDisk compatibility chunk aggregation. - pub const LOCAL_DISK_COMPAT_COLLECT_CHUNKS: &str = "rustfs.io.local_disk.compat_collect.chunks"; + /// Total number of Direct I/O operations + pub const DIRECT_IO_OPERATIONS_TOTAL: &str = "rustfs_direct_io_operations_total"; - /// Byte distribution for LocalDisk compatibility chunk aggregation. - pub const LOCAL_DISK_COMPAT_COLLECT_BYTES: &str = "rustfs.io.local_disk.compat_collect.bytes"; + /// Total bytes processed by Direct I/O + pub const DIRECT_IO_BYTES_TOTAL: &str = "rustfs_direct_io_bytes_total"; - /// Total number of attempted PUT fast paths. - pub const PUT_FAST_PATH_ATTEMPTS_TOTAL: &str = "rustfs.io.put.fast_path.attempts_total"; + /// Average copy count per operation + pub const AVG_COPY_COUNT: &str = "rustfs_zero_copy_avg_copy_count"; - /// Size distribution for attempted PUT fast paths. - pub const PUT_FAST_PATH_ATTEMPT_SIZE_BYTES: &str = "rustfs.io.put.fast_path.attempt.size.bytes"; + /// Throughput in MB/s + pub const THROUGHPUT_MBPS: &str = "rustfs_zero_copy_throughput_mbps"; - /// Total number of transformed PUT selections grouped by transform kind and ingress path. - pub const PUT_TRANSFORM_SELECTED_TOTAL: &str = "rustfs.io.put.transform.selected_total"; - - /// Size distribution for transformed PUT selections. - pub const PUT_TRANSFORM_SIZE_BYTES: &str = "rustfs.io.put.transform.size.bytes"; + /// Memory saved by zero-copy in bytes + pub const MEMORY_SAVED_BYTES: &str = "rustfs_zero_copy_memory_saved_bytes"; } diff --git a/crates/object-io/Cargo.toml b/crates/object-io/Cargo.toml deleted file mode 100644 index 4659f5c9b..000000000 --- a/crates/object-io/Cargo.toml +++ /dev/null @@ -1,36 +0,0 @@ -[package] -name = "rustfs-object-io" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true -homepage.workspace = true -description = "Object I/O policy helpers and zero-copy support primitives for RustFS." -keywords.workspace = true -categories.workspace = true - -[lints] -workspace = true - -[dependencies] -atoi = { workspace = true } -bytes = { workspace = true } -futures-util = { workspace = true } -rustfs-ecstore = { workspace = true } -rustfs-io-core = { workspace = true } -rustfs-io-metrics = { workspace = true } -rustfs-rio = { workspace = true } -rustfs-s3select-api = { workspace = true } -rustfs-utils = { workspace = true ,features = ["http"]} -http = { workspace = true } -s3s.workspace = true -thiserror = { workspace = true } -time = { workspace = true, features = ["parsing", "formatting"] } -tokio = { workspace = true, features = ["io-util"] } -tokio-util = { workspace = true, features = ["io"] } -astral-tokio-tar = { workspace = true } -uuid = { workspace = true } - -[lib] -doctest = false diff --git a/crates/object-io/src/get.rs b/crates/object-io/src/get.rs deleted file mode 100644 index 38a6ec0d9..000000000 --- a/crates/object-io/src/get.rs +++ /dev/null @@ -1,804 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use bytes::Bytes; -use http::HeaderMap; -use http::header::{CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_LANGUAGE}; -use rustfs_ecstore::client::object_api_utils::to_s3s_etag; -use rustfs_ecstore::store_api::{HTTPRangeSpec, ObjectInfo}; -use rustfs_rio::Reader; -use rustfs_s3select_api::object_store::bytes_stream; -use rustfs_utils::http::{AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE}; -use s3s::dto::{ - ChecksumType, ContentType, GetObjectOutput, SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryption, - StreamingBlob, Timestamp, -}; -use std::collections::HashMap; -use std::str::FromStr; -use std::sync::Arc; -#[cfg(test)] -use time::OffsetDateTime; -use tokio::io::{AsyncRead, AsyncSeek, ReadBuf}; -use tokio_util::io::ReaderStream; - -pub struct InMemoryAsyncReader { - cursor: std::io::Cursor, -} - -impl InMemoryAsyncReader { - pub fn new(data: Bytes) -> Self { - Self { - cursor: std::io::Cursor::new(data), - } - } -} - -impl AsyncRead for InMemoryAsyncReader { - fn poll_read( - mut self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> std::task::Poll> { - let unfilled = buf.initialize_unfilled(); - let bytes_read = std::io::Read::read(&mut self.cursor, unfilled)?; - buf.advance(bytes_read); - std::task::Poll::Ready(Ok(())) - } -} - -impl AsyncSeek for InMemoryAsyncReader { - fn start_seek(mut self: std::pin::Pin<&mut Self>, position: std::io::SeekFrom) -> std::io::Result<()> { - std::io::Seek::seek(&mut self.cursor, position)?; - Ok(()) - } - - fn poll_complete(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll> { - std::task::Poll::Ready(Ok(self.cursor.position())) - } -} - -pub fn build_memory_blob(buf: Bytes, response_content_length: i64, optimal_buffer_size: usize) -> Option { - let mem_reader = InMemoryAsyncReader::new(buf); - Some(StreamingBlob::wrap(bytes_stream( - ReaderStream::with_capacity(Box::new(mem_reader), optimal_buffer_size), - response_content_length as usize, - ))) -} - -#[derive(Clone)] -pub struct FrozenGetObjectBody { - body: Arc, -} - -impl FrozenGetObjectBody { - pub fn new(body: Bytes) -> Self { - Self { body: Arc::new(body) } - } - - pub fn shared_body(&self) -> &Arc { - &self.body - } - - pub fn into_shared_body(self) -> Arc { - self.body - } - - pub fn build_blob(&self, response_content_length: i64, optimal_buffer_size: usize) -> Option { - build_memory_blob((*self.body).clone(), response_content_length, optimal_buffer_size) - } -} - -pub fn build_reader_blob(reader: R, response_content_length: i64, optimal_buffer_size: usize) -> Option -where - R: AsyncRead + Send + Sync + 'static, -{ - Some(StreamingBlob::wrap(bytes_stream( - ReaderStream::with_capacity(reader, optimal_buffer_size), - response_content_length as usize, - ))) -} - -pub fn get_object_sequential_hint(rs: Option<&HTTPRangeSpec>) -> bool { - if rs.is_none() { - true - } else if let Some(range_spec) = rs { - range_spec.start == 0 && !range_spec.is_suffix_length - } else { - false - } -} - -pub struct GetObjectOutputContext { - pub output: GetObjectOutput, - pub event_info: ObjectInfo, - pub response_content_length: i64, - pub optimal_buffer_size: usize, - pub copy_mode_override: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct GetObjectStrategyLayout { - pub is_sequential_hint: bool, - pub optimal_buffer_size: usize, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct GetObjectBodyPlanningInputs { - pub is_part_request: bool, - pub is_range_request: bool, - pub encryption_applied: bool, - pub response_size: i64, -} - -pub enum GetObjectBodySource { - Reader(Box), -} - -pub struct GetObjectReadSetup { - pub info: ObjectInfo, - pub event_info: ObjectInfo, - pub body_source: GetObjectBodySource, - pub rs: Option, - pub content_type: Option, - pub last_modified: Option, - pub response_content_length: i64, - pub content_range: Option, - pub server_side_encryption: Option, - pub sse_customer_algorithm: Option, - pub sse_customer_key_md5: Option, - pub ssekms_key_id: Option, - pub encryption_applied: bool, -} - -pub struct LegacyReadPlan { - pub rs: Option, - pub content_type: Option, - pub last_modified: Option, - pub response_content_length: i64, - pub content_range: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GetObjectBodyPlan { - BufferEncrypted, - BufferSeekable, - Stream, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct GetObjectDataPlaneMetricContract { - pub io_path: rustfs_io_metrics::IoPath, - pub copy_mode: rustfs_io_metrics::CopyMode, -} - -impl GetObjectDataPlaneMetricContract { - pub fn disk(io_path: rustfs_io_metrics::IoPath, copy_mode: rustfs_io_metrics::CopyMode) -> Self { - Self { io_path, copy_mode } - } -} - -pub struct GetObjectBodyMaterialization { - pub body: Option, - pub plan: GetObjectBodyPlan, -} - -#[derive(Default)] -pub struct GetObjectEncryptionState { - pub server_side_encryption: Option, - pub sse_customer_algorithm: Option, - pub sse_customer_key_md5: Option, - pub ssekms_key_id: Option, - pub encryption_applied: bool, - pub response_content_length_override: Option, -} - -#[derive(Debug, thiserror::Error)] -pub enum MaterializeGetObjectBodyError { - #[error("failed to read decrypted object: {0}")] - EncryptedRead(std::io::Error), -} - -pub struct GetObjectFlowResult { - pub output: GetObjectOutput, - pub event_info: ObjectInfo, - pub version_id_for_event: String, -} - -pub fn build_get_object_flow_result( - output: GetObjectOutput, - event_info: ObjectInfo, - version_id_for_event: String, -) -> GetObjectFlowResult { - GetObjectFlowResult { - output, - event_info, - version_id_for_event, - } -} - -pub fn build_cors_wrapped_get_object_flow_result( - output_context: GetObjectOutputContext, - version_id_for_event: String, -) -> GetObjectFlowResult { - let GetObjectOutputContext { - output, - event_info, - response_content_length: _, - optimal_buffer_size: _, - copy_mode_override: _, - } = output_context; - build_get_object_flow_result(output, event_info, version_id_for_event) -} - -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct GetObjectChecksums { - pub crc32: Option, - pub crc32c: Option, - pub sha1: Option, - pub sha256: Option, - pub crc64nvme: Option, - pub checksum_type: Option, -} - -fn decode_get_object_checksums(decrypted_checksums: HashMap) -> GetObjectChecksums { - let mut checksums = GetObjectChecksums::default(); - - for (key, checksum) in decrypted_checksums { - if key == AMZ_CHECKSUM_TYPE { - checksums.checksum_type = Some(ChecksumType::from(checksum)); - continue; - } - - match rustfs_rio::ChecksumType::from_string(key.as_str()) { - rustfs_rio::ChecksumType::CRC32 => checksums.crc32 = Some(checksum), - rustfs_rio::ChecksumType::CRC32C => checksums.crc32c = Some(checksum), - rustfs_rio::ChecksumType::SHA1 => checksums.sha1 = Some(checksum), - rustfs_rio::ChecksumType::SHA256 => checksums.sha256 = Some(checksum), - rustfs_rio::ChecksumType::CRC64_NVME => checksums.crc64nvme = Some(checksum), - _ => (), - } - } - - checksums -} - -fn read_object_checksums( - info: &ObjectInfo, - headers: &HeaderMap, - part_number: Option, -) -> std::io::Result { - let (decrypted_checksums, _is_multipart) = info - .decrypt_checksums(part_number.unwrap_or(0), headers) - .map_err(|e| std::io::Error::other(e.to_string()))?; - - Ok(decode_get_object_checksums(decrypted_checksums)) -} - -pub fn build_get_object_checksums( - info: &ObjectInfo, - headers: &HeaderMap, - part_number: Option, - rs: Option<&HTTPRangeSpec>, -) -> std::io::Result { - if let Some(checksum_mode) = headers.get(AMZ_CHECKSUM_MODE) - && checksum_mode.to_str().unwrap_or_default() == "ENABLED" - && rs.is_none() - { - return read_object_checksums(info, headers, part_number); - } - - Ok(GetObjectChecksums::default()) -} - -pub fn build_output_version_id(versioned: bool, version_id: Option<&uuid::Uuid>) -> Option { - if !versioned { - return None; - } - - version_id.map(|vid| { - if *vid == uuid::Uuid::nil() { - "null".to_string() - } else { - vid.to_string() - } - }) -} - -pub fn plan_get_object_strategy_layout( - rs: Option<&HTTPRangeSpec>, - response_content_length: i64, - suggested_buffer_size: usize, - fallback_buffer_size: usize, -) -> GetObjectStrategyLayout { - let is_sequential_hint = get_object_sequential_hint(rs); - let optimal_buffer_size = if suggested_buffer_size > 0 { - suggested_buffer_size.min(fallback_buffer_size) - } else { - rustfs_io_core::get_concurrency_aware_buffer_size(response_content_length, fallback_buffer_size) - }; - - GetObjectStrategyLayout { - is_sequential_hint, - optimal_buffer_size, - } -} - -pub fn plan_get_object_body( - planning_inputs: GetObjectBodyPlanningInputs, - seekable_object_size_threshold: usize, -) -> GetObjectBodyPlan { - let should_buffer_for_seek = planning_inputs.response_size > 0 - && planning_inputs.response_size <= seekable_object_size_threshold as i64 - && !planning_inputs.is_part_request - && !planning_inputs.is_range_request; - - if planning_inputs.encryption_applied && should_buffer_for_seek { - GetObjectBodyPlan::BufferEncrypted - } else if should_buffer_for_seek { - GetObjectBodyPlan::BufferSeekable - } else { - GetObjectBodyPlan::Stream - } -} - -pub async fn materialize_get_object_body( - mut final_stream: R, - plan: GetObjectBodyPlan, - response_content_length: i64, - optimal_buffer_size: usize, -) -> Result -where - R: AsyncRead + Send + Sync + Unpin + 'static, -{ - match plan { - GetObjectBodyPlan::BufferEncrypted => { - let mut buf = Vec::with_capacity(response_content_length as usize); - tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf) - .await - .map_err(MaterializeGetObjectBodyError::EncryptedRead)?; - let body = FrozenGetObjectBody::new(Bytes::from(buf)); - - Ok(GetObjectBodyMaterialization { - body: body.build_blob(response_content_length, optimal_buffer_size), - plan, - }) - } - GetObjectBodyPlan::BufferSeekable => { - let mut buf = Vec::with_capacity(response_content_length as usize); - let body = match tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { - Ok(_) => FrozenGetObjectBody::new(Bytes::from(buf)).build_blob(response_content_length, optimal_buffer_size), - Err(_) => build_reader_blob(final_stream, response_content_length, optimal_buffer_size), - }; - - Ok(GetObjectBodyMaterialization { body, plan }) - } - GetObjectBodyPlan::Stream => Ok(GetObjectBodyMaterialization { - body: build_reader_blob(final_stream, response_content_length, optimal_buffer_size), - plan, - }), - } -} - -fn resolve_requested_range( - info: &ObjectInfo, - mut rs: Option, - part_number: Option, -) -> Option { - if let Some(part_number) = part_number - && rs.is_none() - { - rs = HTTPRangeSpec::from_object_info(info, part_number); - } - - rs -} - -fn resolve_response_range( - total_size: i64, - rs: Option, -) -> std::io::Result<(Option, i64, Option)> { - let Some(range_spec) = rs else { - return Ok((None, total_size, None)); - }; - - let (start, length) = range_spec.get_offset_length(total_size)?; - let content_range = Some(format!("bytes {}-{}/{}", start, start as i64 + length - 1, total_size)); - - Ok((Some(range_spec), length, content_range)) -} - -pub fn plan_legacy_read( - info: &ObjectInfo, - rs: Option, - part_number: Option, -) -> std::io::Result { - let content_type = info - .content_type - .as_ref() - .and_then(|content_type| ContentType::from_str(content_type).ok()); - let last_modified = info.mod_time.map(Timestamp::from); - let rs = resolve_requested_range(info, rs, part_number); - let total_size = info.get_actual_size()?; - let (rs, response_content_length, content_range) = resolve_response_range(total_size, rs)?; - - Ok(LegacyReadPlan { - rs, - content_type, - last_modified, - response_content_length, - content_range, - }) -} - -pub fn build_reader_read_setup( - info: ObjectInfo, - event_info: ObjectInfo, - final_stream: Box, - plan: LegacyReadPlan, - encryption_state: GetObjectEncryptionState, -) -> GetObjectReadSetup { - let LegacyReadPlan { - rs, - content_type, - last_modified, - response_content_length, - content_range, - } = plan; - - let GetObjectEncryptionState { - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, - encryption_applied, - response_content_length_override, - } = encryption_state; - - GetObjectReadSetup { - info, - event_info, - body_source: GetObjectBodySource::Reader(final_stream), - rs, - content_type, - last_modified, - response_content_length: response_content_length_override.unwrap_or(response_content_length), - content_range, - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, - encryption_applied, - } -} - -#[allow(clippy::too_many_arguments)] -pub fn build_get_object_output_context( - body: Option, - info: ObjectInfo, - event_info: ObjectInfo, - content_type: Option, - last_modified: Option, - response_content_length: i64, - content_range: Option, - server_side_encryption: Option, - sse_customer_algorithm: Option, - sse_customer_key_md5: Option, - ssekms_key_id: Option, - checksums: &GetObjectChecksums, - filtered_metadata: Option>, - versioned: bool, - optimal_buffer_size: usize, - copy_mode_override: Option, -) -> GetObjectOutputContext { - let output_version_id = build_output_version_id(versioned, info.version_id.as_ref()); - let output = build_get_object_output( - body, - &info, - content_type, - last_modified, - response_content_length, - content_range, - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, - checksums, - output_version_id, - filtered_metadata, - ); - - GetObjectOutputContext { - output, - event_info, - response_content_length, - optimal_buffer_size, - copy_mode_override, - } -} - -#[allow(clippy::too_many_arguments)] -pub fn build_get_object_output( - body: Option, - info: &ObjectInfo, - content_type: Option, - last_modified: Option, - response_content_length: i64, - content_range: Option, - server_side_encryption: Option, - sse_customer_algorithm: Option, - sse_customer_key_md5: Option, - ssekms_key_id: Option, - checksums: &GetObjectChecksums, - output_version_id: Option, - filtered_metadata: Option>, -) -> GetObjectOutput { - GetObjectOutput { - body, - content_length: Some(response_content_length), - last_modified, - expires: info.expires.map(Timestamp::from), - content_type, - cache_control: info.user_defined.get(CACHE_CONTROL.as_str()).cloned(), - content_disposition: info.user_defined.get(CONTENT_DISPOSITION.as_str()).cloned(), - content_encoding: info.content_encoding.clone(), - content_language: info.user_defined.get(CONTENT_LANGUAGE.as_str()).cloned(), - accept_ranges: Some("bytes".to_string()), - content_range, - e_tag: info.etag.as_ref().map(|etag| to_s3s_etag(etag)), - metadata: filtered_metadata, - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, - checksum_crc32: checksums.crc32.clone(), - checksum_crc32c: checksums.crc32c.clone(), - checksum_sha1: checksums.sha1.clone(), - checksum_sha256: checksums.sha256.clone(), - checksum_crc64nvme: checksums.crc64nvme.clone(), - checksum_type: checksums.checksum_type.clone(), - version_id: output_version_id, - ..Default::default() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn build_get_object_checksums_returns_default_when_mode_absent() { - let checksums = build_get_object_checksums(&ObjectInfo::default(), &HeaderMap::new(), None, None).unwrap(); - assert_eq!(checksums, GetObjectChecksums::default()); - } - - #[test] - fn plan_legacy_read_uses_part_number_range_when_available() { - let mut info = ObjectInfo { - size: 12, - actual_size: 12, - content_type: Some("application/octet-stream".to_string()), - ..Default::default() - }; - info.parts = vec![Default::default(), Default::default()]; - info.parts[0].number = 1; - info.parts[0].size = 5; - info.parts[0].actual_size = 5; - info.parts[1].number = 2; - info.parts[1].size = 7; - info.parts[1].actual_size = 7; - - let plan = plan_legacy_read(&info, None, Some(2)).unwrap(); - - let rs = plan.rs.expect("range from part number"); - assert_eq!(rs.start, 5); - assert_eq!(rs.end, 11); - assert_eq!(plan.response_content_length, 7); - assert_eq!(plan.content_range.as_deref(), Some("bytes 5-11/12")); - assert!(plan.content_type.is_some()); - } - - #[test] - fn build_reader_read_setup_uses_encryption_length_override() { - let plan = LegacyReadPlan { - rs: None, - content_type: None, - last_modified: None, - response_content_length: 16, - content_range: None, - }; - let encryption_state = GetObjectEncryptionState { - encryption_applied: true, - response_content_length_override: Some(12), - ..Default::default() - }; - let reader = Box::new(rustfs_rio::WarpReader::new(tokio::io::empty())) as Box; - - let setup = build_reader_read_setup(ObjectInfo::default(), ObjectInfo::default(), reader, plan, encryption_state); - - assert!(setup.encryption_applied); - assert_eq!(setup.response_content_length, 12); - match setup.body_source { - GetObjectBodySource::Reader(_) => {} - } - } - - #[test] - fn plan_get_object_body_buffers_seekable_small_plain_request() { - let plan = plan_get_object_body( - GetObjectBodyPlanningInputs { - is_part_request: false, - is_range_request: false, - encryption_applied: false, - response_size: 1024, - }, - 4096, - ); - - assert_eq!(plan, GetObjectBodyPlan::BufferSeekable); - } - - #[test] - fn disk_metric_contract_preserves_io_labels() { - let contract = - GetObjectDataPlaneMetricContract::disk(rustfs_io_metrics::IoPath::Legacy, rustfs_io_metrics::CopyMode::SingleCopy); - - assert_eq!(contract.io_path, rustfs_io_metrics::IoPath::Legacy); - assert_eq!(contract.copy_mode, rustfs_io_metrics::CopyMode::SingleCopy); - } - - #[test] - fn plan_get_object_body_uses_encrypted_buffer_for_small_plain_request() { - let plan = plan_get_object_body( - GetObjectBodyPlanningInputs { - is_part_request: false, - is_range_request: false, - encryption_applied: true, - response_size: 1024, - }, - 4096, - ); - - assert_eq!(plan, GetObjectBodyPlan::BufferEncrypted); - } - - #[test] - fn get_object_sequential_hint_distinguishes_prefix_and_suffix_ranges() { - assert!(get_object_sequential_hint(None)); - assert!(get_object_sequential_hint(Some(&HTTPRangeSpec { - is_suffix_length: false, - start: 0, - end: -1, - }))); - assert!(!get_object_sequential_hint(Some(&HTTPRangeSpec { - is_suffix_length: false, - start: 4, - end: 8, - }))); - assert!(!get_object_sequential_hint(Some(&HTTPRangeSpec { - is_suffix_length: true, - start: 4, - end: -1, - }))); - } - - #[test] - fn plan_get_object_strategy_layout_caps_buffer_to_fallback() { - let layout = plan_get_object_strategy_layout(None, 1024, 8192, 4096); - - assert!(layout.is_sequential_hint); - assert_eq!(layout.optimal_buffer_size, 4096); - } - - #[test] - fn frozen_get_object_body_reuses_same_shared_bytes_for_memory_blob() { - let frozen = FrozenGetObjectBody::new(Bytes::from_static(b"abc")); - let shared = Arc::clone(frozen.shared_body()); - assert_eq!(*shared, Bytes::from_static(b"abc")); - assert!(Arc::ptr_eq(&shared, frozen.shared_body())); - } - - #[test] - fn build_output_version_id_maps_nil_uuid_to_null() { - let nil = uuid::Uuid::nil(); - let version_id = build_output_version_id(true, Some(&nil)); - - assert_eq!(version_id.as_deref(), Some("null")); - } - - #[test] - fn build_get_object_output_context_preserves_copy_mode_override() { - let info = ObjectInfo { - version_id: Some(uuid::Uuid::nil()), - ..Default::default() - }; - let output_context = build_get_object_output_context( - None, - info, - ObjectInfo::default(), - None, - None, - 8, - None, - None, - None, - None, - None, - &GetObjectChecksums::default(), - None, - true, - 4096, - Some(rustfs_io_metrics::CopyMode::Reconstructed), - ); - - assert_eq!(output_context.output.version_id.as_deref(), Some("null")); - assert_eq!(output_context.copy_mode_override, Some(rustfs_io_metrics::CopyMode::Reconstructed)); - assert_eq!(output_context.optimal_buffer_size, 4096); - } - - #[test] - fn build_get_object_output_preserves_http_metadata_like_cached_path() { - let mut info = ObjectInfo { - content_type: Some("application/octet-stream".to_string()), - content_encoding: Some("zstd".to_string()), - etag: Some("etag".to_string()), - expires: Some(OffsetDateTime::UNIX_EPOCH), - ..Default::default() - }; - info.user_defined - .insert("cache-control".to_string(), "max-age=3600".to_string()); - info.user_defined - .insert("content-disposition".to_string(), "attachment; filename=\"bundle.zip\"".to_string()); - info.user_defined.insert("content-language".to_string(), "en-US".to_string()); - - let output = build_get_object_output( - None, - &info, - info.content_type - .as_ref() - .and_then(|content_type| ContentType::from_str(content_type).ok()), - info.mod_time.map(Timestamp::from), - 8, - None, - None, - None, - None, - None, - &GetObjectChecksums::default(), - None, - None, - ); - - assert_eq!(output.cache_control.as_deref(), Some("max-age=3600")); - assert_eq!(output.content_disposition.as_deref(), Some("attachment; filename=\"bundle.zip\"")); - assert_eq!(output.content_language.as_deref(), Some("en-US")); - assert_eq!(output.content_encoding.as_deref(), Some("zstd")); - assert_eq!(output.expires, Some(Timestamp::from(OffsetDateTime::UNIX_EPOCH))); - } - - #[test] - fn build_cors_wrapped_get_object_flow_result_uses_wrapped_mode() { - let result = build_cors_wrapped_get_object_flow_result( - GetObjectOutputContext { - output: GetObjectOutput::default(), - event_info: ObjectInfo::default(), - response_content_length: 1, - optimal_buffer_size: 1024, - copy_mode_override: None, - }, - "vid".to_string(), - ); - - assert_eq!(result.version_id_for_event, "vid"); - } -} diff --git a/crates/object-io/src/lib.rs b/crates/object-io/src/lib.rs deleted file mode 100644 index 9de88dbe3..000000000 --- a/crates/object-io/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod get; -pub mod put; diff --git a/crates/object-io/src/put.rs b/crates/object-io/src/put.rs deleted file mode 100644 index 15266ee6a..000000000 --- a/crates/object-io/src/put.rs +++ /dev/null @@ -1,1439 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use bytes::Buf; -use futures_util::{Stream, StreamExt}; -use http::HeaderMap; -use rustfs_ecstore::compress::{MIN_COMPRESSIBLE_SIZE, is_compressible}; -use rustfs_ecstore::store_api::ObjectOptions; -use rustfs_rio::{Checksum, EtagResolvable, HashReader, HashReaderDetector, Reader, TryGetIndex, WarpReader}; -use rustfs_utils::http::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM; -use rustfs_utils::http::headers::{ - AMZ_DECODED_CONTENT_LENGTH, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, AMZ_MINIO_SNOWBALL_PREFIX, - AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, AMZ_RUSTFS_SNOWBALL_PREFIX, AMZ_SERVER_SIDE_ENCRYPTION, - AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_SNOWBALL_EXTRACT, AMZ_SNOWBALL_IGNORE_DIRS, AMZ_SNOWBALL_IGNORE_ERRORS, - AMZ_SNOWBALL_PREFIX, -}; -use s3s::dto::{ChecksumAlgorithm, PutObjectInput, ServerSideEncryption}; -use s3s::{S3Error, s3_error}; -use std::collections::HashMap; -use std::pin::Pin; -use tokio::io::AsyncRead; -use tokio_tar::Archive; -use tokio_util::io::StreamReader; - -pub const AMZ_SNOWBALL_EXTRACT_COMPAT: &str = "X-Amz-Snowball-Auto-Extract"; -pub const AMZ_SNOWBALL_PREFIX_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix"; -pub const AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Dirs"; -pub const AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Errors"; - -const AMZ_META_PREFIX_LOWER: &str = "x-amz-meta-"; -const SNOWBALL_PREFIX_SUFFIX_LOWER: &str = "snowball-prefix"; -const SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER: &str = "snowball-ignore-dirs"; -const SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER: &str = "snowball-ignore-errors"; -const SNOWBALL_PREFIX_HEADER_KEYS: &[&str] = &[AMZ_MINIO_SNOWBALL_PREFIX, AMZ_SNOWBALL_PREFIX, AMZ_RUSTFS_SNOWBALL_PREFIX]; -const SNOWBALL_IGNORE_DIRS_HEADER_KEYS: &[&str] = &[ - AMZ_MINIO_SNOWBALL_IGNORE_DIRS, - AMZ_SNOWBALL_IGNORE_DIRS, - AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, -]; -const SNOWBALL_IGNORE_ERRORS_HEADER_KEYS: &[&str] = &[ - AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, - AMZ_SNOWBALL_IGNORE_ERRORS, - AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, -]; -pub const PUT_REDUCED_COPY_MIN_SIZE_BYTES: i64 = 1024 * 1024; - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct PutObjectChecksums { - pub crc32: Option, - pub crc32c: Option, - pub sha1: Option, - pub sha256: Option, - pub crc64nvme: Option, -} - -impl PutObjectChecksums { - pub fn merge_from_map(&mut self, checksums: &HashMap) { - for (key, checksum) in checksums { - match rustfs_rio::ChecksumType::from_string(key.as_str()) { - rustfs_rio::ChecksumType::CRC32 => self.crc32 = Some(checksum.clone()), - rustfs_rio::ChecksumType::CRC32C => self.crc32c = Some(checksum.clone()), - rustfs_rio::ChecksumType::SHA1 => self.sha1 = Some(checksum.clone()), - rustfs_rio::ChecksumType::SHA256 => self.sha256 = Some(checksum.clone()), - rustfs_rio::ChecksumType::CRC64_NVME => self.crc64nvme = Some(checksum.clone()), - _ => {} - } - } - } -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct PutObjectTransformStage { - compression_applied: bool, - encryption_applied: bool, -} - -impl PutObjectTransformStage { - pub fn mark_compression(&mut self) { - self.compression_applied = true; - } - - pub fn mark_encryption(&mut self) { - self.encryption_applied = true; - } - - #[must_use] - pub const fn compression_applied(self) -> bool { - self.compression_applied - } - - #[must_use] - pub const fn encryption_applied(self) -> bool { - self.encryption_applied - } - - #[must_use] - pub fn effective_copy_mode(self) -> rustfs_io_metrics::CopyMode { - resolve_put_effective_copy_mode(self.compression_applied, self.encryption_applied) - } - - #[must_use] - pub fn metric_kind(self) -> Option<&'static str> { - resolve_put_transform_metric_kind(self.compression_applied, self.encryption_applied) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PutObjectCompatIngressKind { - BufferedStreamCompat, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PutObjectCompatIngressPlan { - pub kind: PutObjectCompatIngressKind, - pub buffer_size: usize, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PutObjectIngressKind { - LegacyCompat, - ReducedCopyCandidate, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PutObjectIngressPlan { - pub kind: PutObjectIngressKind, - pub compat: PutObjectCompatIngressPlan, - pub enable_zero_copy: bool, -} - -pub type BoxIngressStream = Pin> + Send + Sync + 'static>>; -pub type PutObjectCompatIngressStream = futures_util::stream::Map) -> std::io::Result>; -pub type PutObjectCompatIngress = tokio::io::BufReader, B>>; - -pub fn box_put_object_ingress_stream(body: S) -> BoxIngressStream -where - S: Stream> + Send + Sync + 'static, -{ - Box::pin(body) -} - -pub struct PutObjectReducedCopyIngress { - body: BoxIngressStream, - compat: PutObjectCompatIngressPlan, -} - -impl PutObjectReducedCopyIngress { - pub const fn new(body: BoxIngressStream, compat: PutObjectCompatIngressPlan) -> Self { - Self { body, compat } - } - - pub fn from_stream(body: S, compat: PutObjectCompatIngressPlan) -> Self - where - S: Stream> + Send + Sync + 'static, - { - Self::new(box_put_object_ingress_stream(body), compat) - } - - pub const fn compat_plan(&self) -> PutObjectCompatIngressPlan { - self.compat - } - - pub fn into_body(self) -> BoxIngressStream { - self.body - } -} - -impl PutObjectReducedCopyIngress -where - B: Buf, - E: std::fmt::Display, - PutObjectCompatIngress, B, E>: Send + Sync + Unpin + 'static, -{ - pub fn into_compat_reader(self) -> Box { - build_put_object_compat_reader(self.body, self.compat) - } -} - -pub enum PutObjectIngressSource { - LegacyCompat(Box), - ReducedCopyCandidate(PutObjectReducedCopyIngress), -} - -struct PutObjectReducedCopyReader { - body: S, - current_chunk: Option, - pending_error: Option, -} - -impl PutObjectReducedCopyReader { - fn new(body: S) -> Self { - Self { - body, - current_chunk: None, - pending_error: None, - } - } - - fn copy_chunk_into_slice(chunk: &mut B, buf: &mut [u8]) -> usize - where - B: Buf, - { - let mut copied = 0; - let to_copy = chunk.remaining().min(buf.len()); - - while copied < to_copy { - let slice = chunk.chunk(); - if !slice.is_empty() { - let len = (to_copy - copied).min(slice.len()); - buf[copied..copied + len].copy_from_slice(&slice[..len]); - chunk.advance(len); - copied += len; - continue; - } - - let dest = &mut buf[copied..to_copy]; - chunk.copy_to_slice(dest); - copied = to_copy; - } - - copied - } - - fn copy_chunk_into_read_buf(chunk: &mut B, buf: &mut tokio::io::ReadBuf<'_>) -> usize - where - B: Buf, - { - let to_copy = chunk.remaining().min(buf.remaining()); - let dest = &mut buf.initialize_unfilled()[..to_copy]; - let copied = Self::copy_chunk_into_slice(chunk, dest); - buf.advance(copied); - copied - } -} - -impl AsyncRead for PutObjectReducedCopyReader -where - S: Stream> + Unpin, - B: Buf + Unpin, - E: std::fmt::Display, -{ - fn poll_read( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - buf: &mut tokio::io::ReadBuf<'_>, - ) -> std::task::Poll> { - let this = self.get_mut(); - - if let Some(err) = this.pending_error.take() { - return std::task::Poll::Ready(Err(err)); - } - - let mut read_any = false; - - loop { - if buf.remaining() == 0 { - return std::task::Poll::Ready(Ok(())); - } - - if let Some(chunk) = this.current_chunk.as_mut() { - if !chunk.has_remaining() { - this.current_chunk = None; - continue; - } - - if Self::copy_chunk_into_read_buf(chunk, buf) > 0 { - read_any = true; - } - - if chunk.has_remaining() || buf.remaining() == 0 { - return std::task::Poll::Ready(Ok(())); - } - - this.current_chunk = None; - continue; - } - - match std::pin::Pin::new(&mut this.body).poll_next(cx) { - std::task::Poll::Ready(Some(Ok(chunk))) => { - if chunk.remaining() == 0 { - continue; - } - this.current_chunk = Some(chunk); - } - std::task::Poll::Ready(Some(Err(err))) => { - let err = std::io::Error::other(err.to_string()); - if read_any { - this.pending_error = Some(err); - return std::task::Poll::Ready(Ok(())); - } - return std::task::Poll::Ready(Err(err)); - } - std::task::Poll::Ready(None) => return std::task::Poll::Ready(Ok(())), - std::task::Poll::Pending => { - if read_any { - return std::task::Poll::Ready(Ok(())); - } - return std::task::Poll::Pending; - } - } - } - } -} - -impl EtagResolvable for PutObjectReducedCopyReader {} - -impl HashReaderDetector for PutObjectReducedCopyReader {} - -impl TryGetIndex for PutObjectReducedCopyReader {} - -fn build_put_object_reduced_copy_reader(candidate: PutObjectReducedCopyIngress) -> Box -where - B: Buf + Send + Sync + Unpin + 'static, - E: std::fmt::Display + Send + Sync + 'static, -{ - Box::new(PutObjectReducedCopyReader::new(candidate.into_body())) -} - -impl PutObjectIngressSource -where - B: Buf, - E: std::fmt::Display, - PutObjectCompatIngress, B, E>: Send + Sync + Unpin + 'static, -{ - pub fn into_compat_reader(self) -> Box { - match self { - Self::LegacyCompat(reader) => reader, - Self::ReducedCopyCandidate(candidate) => candidate.into_compat_reader(), - } - } - - pub fn into_reader(self) -> Box - where - B: Buf + Send + Sync + Unpin + 'static, - E: std::fmt::Display + Send + Sync + 'static, - { - match self { - Self::LegacyCompat(reader) => reader, - Self::ReducedCopyCandidate(candidate) => build_put_object_reduced_copy_reader(candidate), - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PutObjectPlainBodyKind { - LegacyCompat, - ReducedCopyCandidate, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PutObjectBodyKind { - Plain(PutObjectPlainBodyKind), - Compressed, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PutObjectBodyPlan { - pub ingress: PutObjectIngressPlan, - pub kind: PutObjectBodyKind, -} - -impl PutObjectBodyPlan { - pub const fn should_compress(&self) -> bool { - matches!(self.kind, PutObjectBodyKind::Compressed) - } - - pub const fn plain_body_kind(&self) -> Option { - match self.kind { - PutObjectBodyKind::Plain(kind) => Some(kind), - PutObjectBodyKind::Compressed => None, - } - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct PutObjectExtractOptions { - pub prefix: Option, - pub ignore_dirs: bool, - pub ignore_errors: bool, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct PutObjectLegacyHashValues { - pub md5hex: Option, - pub sha256hex: Option, -} - -impl PutObjectLegacyHashValues { - pub fn clear_for_transformed_body(&mut self) { - self.md5hex = None; - self.sha256hex = None; - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PutObjectLegacyHashStagePlan { - pub size: i64, - pub actual_size: i64, - pub apply_s3_checksum: bool, - pub ignore_s3_checksum_value: bool, -} - -pub struct PutObjectHashStage { - pub reader: HashReader, - pub want_checksum: Option, - pub ingress_kind: PutObjectIngressKind, -} - -fn build_put_object_hash_stage( - reader: Box, - ingress_kind: PutObjectIngressKind, - hash_values: PutObjectLegacyHashValues, - plan: PutObjectLegacyHashStagePlan, - headers: &HeaderMap, - trailing_headers: Option, -) -> std::io::Result { - let mut reader = HashReader::new(reader, plan.size, plan.actual_size, hash_values.md5hex, hash_values.sha256hex, false)?; - let requested_checksum_type = rustfs_rio::ChecksumType::from_header(headers); - let want_checksum = if plan.apply_s3_checksum { - reader.add_checksum_from_s3s(headers, trailing_headers, plan.ignore_s3_checksum_value)?; - if requested_checksum_type.is_set() && reader.checksum().is_none() { - reader.enable_auto_checksum(requested_checksum_type)?; - } - reader.checksum() - } else { - None - }; - - Ok(PutObjectHashStage { - reader, - want_checksum, - ingress_kind, - }) -} - -pub fn apply_trailing_checksums( - algorithm: Option<&str>, - trailing_headers: &Option, - checksums: &mut PutObjectChecksums, -) { - let Some(alg) = algorithm else { return }; - let Some(checksum_str) = trailing_headers.as_ref().and_then(|trailer| { - let key = match alg { - ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(), - ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(), - ChecksumAlgorithm::SHA1 => rustfs_rio::ChecksumType::SHA1.key(), - ChecksumAlgorithm::SHA256 => rustfs_rio::ChecksumType::SHA256.key(), - ChecksumAlgorithm::CRC64NVME => rustfs_rio::ChecksumType::CRC64_NVME.key(), - _ => return None, - }; - trailer.read(|headers| { - headers - .get(key.unwrap_or_default()) - .and_then(|value| value.to_str().ok().map(|s| s.to_string())) - }) - }) else { - return; - }; - - match alg { - ChecksumAlgorithm::CRC32 => checksums.crc32 = checksum_str, - ChecksumAlgorithm::CRC32C => checksums.crc32c = checksum_str, - ChecksumAlgorithm::SHA1 => checksums.sha1 = checksum_str, - ChecksumAlgorithm::SHA256 => checksums.sha256 = checksum_str, - ChecksumAlgorithm::CRC64NVME => checksums.crc64nvme = checksum_str, - _ => (), - } -} - -pub fn resolve_put_body_size(content_length: Option, headers: &HeaderMap) -> s3s::S3Result { - let size = match content_length { - Some(c) => c, - None => { - if let Some(val) = headers.get(AMZ_DECODED_CONTENT_LENGTH) { - match atoi::atoi::(val.as_bytes()) { - Some(x) => x, - None => return Err(s3_error!(UnexpectedContent)), - } - } else { - return Err(s3_error!(UnexpectedContent)); - } - } - }; - - if size == -1 { - return Err(s3_error!(UnexpectedContent)); - } - - Ok(size) -} - -pub fn should_use_zero_copy(size: i64, headers: &HeaderMap) -> bool { - const ZERO_COPY_MIN_SIZE: i64 = 1024 * 1024; - - if size <= ZERO_COPY_MIN_SIZE { - return false; - } - - !has_put_encryption_headers(headers) && !put_request_is_compressible(headers) -} - -fn has_put_encryption_headers(headers: &HeaderMap) -> bool { - headers.get(AMZ_SERVER_SIDE_ENCRYPTION).is_some() - || headers.get(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM).is_some() - || headers.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID).is_some() -} - -fn put_request_is_compressible(headers: &HeaderMap) -> bool { - if let Some(content_type) = headers.get("content-type") - && let Ok(ct) = content_type.to_str() - { - let compressible_types = [ - "text/plain", - "text/html", - "text/css", - "text/javascript", - "application/javascript", - "application/json", - "application/xml", - "text/xml", - ]; - return compressible_types.iter().any(|ty| ct.contains(ty)); - } - - false -} - -fn should_use_put_reduced_copy_candidate( - size: i64, - headers: &HeaderMap, - encryption_enabled: bool, - compression_enabled: bool, -) -> bool { - if size <= PUT_REDUCED_COPY_MIN_SIZE_BYTES { - return false; - } - - if !encryption_enabled && has_put_encryption_headers(headers) { - return false; - } - - compression_enabled || !put_request_is_compressible(headers) -} - -fn map_put_object_ingress_error(result: Result) -> std::io::Result -where - E: std::fmt::Display, -{ - result.map_err(|err| std::io::Error::other(err.to_string())) -} - -pub fn build_put_object_compat_ingress(body: S, plan: PutObjectCompatIngressPlan) -> PutObjectCompatIngress -where - S: Stream>, - B: Buf, - E: std::fmt::Display, -{ - match plan.kind { - PutObjectCompatIngressKind::BufferedStreamCompat => tokio::io::BufReader::with_capacity( - plan.buffer_size, - StreamReader::new(body.map(map_put_object_ingress_error:: as fn(Result) -> std::io::Result)), - ), - } -} - -pub fn build_put_object_compat_reader(body: S, plan: PutObjectCompatIngressPlan) -> Box -where - S: Stream>, - B: Buf, - E: std::fmt::Display, - PutObjectCompatIngress: Send + Sync + Unpin + 'static, -{ - Box::new(WarpReader::new(build_put_object_compat_ingress(body, plan))) -} - -pub fn build_put_object_ingress_source(body: S, plan: PutObjectBodyPlan) -> PutObjectIngressSource -where - S: Stream> + Send + Sync + 'static, - B: Buf + 'static, - E: std::fmt::Display + 'static, - PutObjectCompatIngress, B, E>: Send + Sync + Unpin + 'static, -{ - match (plan.kind, plan.ingress.kind) { - (PutObjectBodyKind::Compressed, PutObjectIngressKind::ReducedCopyCandidate) - | (PutObjectBodyKind::Plain(PutObjectPlainBodyKind::ReducedCopyCandidate), PutObjectIngressKind::ReducedCopyCandidate) => { - PutObjectIngressSource::ReducedCopyCandidate(PutObjectReducedCopyIngress::from_stream(body, plan.ingress.compat)) - } - (PutObjectBodyKind::Compressed, _) - | (PutObjectBodyKind::Plain(PutObjectPlainBodyKind::LegacyCompat), _) - | (PutObjectBodyKind::Plain(PutObjectPlainBodyKind::ReducedCopyCandidate), PutObjectIngressKind::LegacyCompat) => { - PutObjectIngressSource::LegacyCompat(build_put_object_compat_reader( - box_put_object_ingress_stream(body), - plan.ingress.compat, - )) - } - } -} - -pub fn build_put_object_legacy_hash_stage( - reader: Box, - hash_values: PutObjectLegacyHashValues, - plan: PutObjectLegacyHashStagePlan, - headers: &HeaderMap, - trailing_headers: Option, -) -> std::io::Result { - build_put_object_hash_stage(reader, PutObjectIngressKind::LegacyCompat, hash_values, plan, headers, trailing_headers) -} - -pub fn build_put_object_plain_hash_stage( - ingress: PutObjectIngressSource, - hash_values: PutObjectLegacyHashValues, - plan: PutObjectLegacyHashStagePlan, - headers: &HeaderMap, - trailing_headers: Option, -) -> std::io::Result -where - B: Buf + Send + Sync + Unpin + 'static, - E: std::fmt::Display + Send + Sync + 'static, -{ - match ingress { - PutObjectIngressSource::LegacyCompat(reader) => { - build_put_object_hash_stage(reader, PutObjectIngressKind::LegacyCompat, hash_values, plan, headers, trailing_headers) - } - PutObjectIngressSource::ReducedCopyCandidate(candidate) => build_put_object_hash_stage( - build_put_object_reduced_copy_reader(candidate), - PutObjectIngressKind::ReducedCopyCandidate, - hash_values, - plan, - headers, - trailing_headers, - ), - } -} - -pub fn plan_put_object_ingress(size: i64, headers: &HeaderMap, buffer_size: usize) -> PutObjectIngressPlan { - plan_put_object_ingress_with_transforms(size, headers, buffer_size, false, false) -} - -pub fn plan_put_object_ingress_with_transforms( - size: i64, - headers: &HeaderMap, - buffer_size: usize, - encryption_enabled: bool, - compression_enabled: bool, -) -> PutObjectIngressPlan { - let enable_zero_copy = should_use_put_reduced_copy_candidate(size, headers, encryption_enabled, compression_enabled); - PutObjectIngressPlan { - kind: if enable_zero_copy { - PutObjectIngressKind::ReducedCopyCandidate - } else { - PutObjectIngressKind::LegacyCompat - }, - compat: PutObjectCompatIngressPlan { - kind: PutObjectCompatIngressKind::BufferedStreamCompat, - buffer_size, - }, - enable_zero_copy, - } -} - -pub fn plan_put_object_body(size: i64, headers: &HeaderMap, key: &str, buffer_size: usize) -> PutObjectBodyPlan { - plan_put_object_body_with_transforms(size, headers, key, buffer_size, false) -} - -pub fn plan_put_object_body_with_transforms( - size: i64, - headers: &HeaderMap, - key: &str, - buffer_size: usize, - encryption_enabled: bool, -) -> PutObjectBodyPlan { - let compression_enabled = size > MIN_COMPRESSIBLE_SIZE as i64 && is_compressible(headers, key); - let ingress = plan_put_object_ingress_with_transforms(size, headers, buffer_size, encryption_enabled, compression_enabled); - let kind = if compression_enabled { - PutObjectBodyKind::Compressed - } else { - PutObjectBodyKind::Plain(match ingress.kind { - PutObjectIngressKind::LegacyCompat => PutObjectPlainBodyKind::LegacyCompat, - PutObjectIngressKind::ReducedCopyCandidate => PutObjectPlainBodyKind::ReducedCopyCandidate, - }) - }; - - PutObjectBodyPlan { ingress, kind } -} - -pub fn resolve_put_effective_copy_mode(applied_compression: bool, applied_encryption: bool) -> rustfs_io_metrics::CopyMode { - if applied_compression || applied_encryption { - rustfs_io_metrics::CopyMode::Transformed - } else { - rustfs_io_metrics::CopyMode::SingleCopy - } -} - -pub fn resolve_put_transform_metric_kind(applied_compression: bool, applied_encryption: bool) -> Option<&'static str> { - match (applied_compression, applied_encryption) { - (true, true) => Some("compression_encryption"), - (true, false) => Some("compression"), - (false, true) => Some("encryption"), - (false, false) => None, - } -} - -pub fn resolve_put_transformed_fallback_reason( - ingress_kind: PutObjectIngressKind, - compressed: bool, - encryption_enabled: bool, -) -> Option { - if ingress_kind == PutObjectIngressKind::ReducedCopyCandidate { - return None; - } - - match (compressed, encryption_enabled) { - (true, true) => Some(rustfs_io_metrics::FallbackReason::TransformCompressionEncryptionLegacy), - (true, false) => Some(rustfs_io_metrics::FallbackReason::TransformCompressionLegacy), - (false, true) => Some(rustfs_io_metrics::FallbackReason::TransformEncryptionLegacy), - (false, false) => None, - } -} - -pub fn header_value_is_true(headers: &HeaderMap, key: &str) -> bool { - headers - .get(key) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.trim().eq_ignore_ascii_case("true")) -} - -pub fn is_put_object_extract_requested(headers: &HeaderMap) -> bool { - header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_COMPAT) -} - -fn trimmed_header_value(headers: &HeaderMap, key: &str) -> Option { - headers - .get(key) - .and_then(|value| value.to_str().ok()) - .map(|value| value.trim().to_string()) -} - -fn is_exact_snowball_meta_key(key: &str, exact_keys: &[&str]) -> bool { - exact_keys.iter().any(|exact_key| key.eq_ignore_ascii_case(exact_key)) -} - -fn snowball_meta_value_by_suffix(headers: &HeaderMap, suffix_lower: &str, exact_keys: &[&str]) -> Option { - for (name, value) in headers { - let key = name.as_str(); - if key.starts_with(AMZ_META_PREFIX_LOWER) - && key.ends_with(suffix_lower) - && !is_exact_snowball_meta_key(key, exact_keys) - && let Ok(parsed) = value.to_str() - { - return Some(parsed.trim().to_string()); - } - } - - None -} - -fn snowball_meta_value(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> Option { - for key in exact_keys { - if let Some(value) = trimmed_header_value(headers, key) { - return Some(value); - } - } - - snowball_meta_value_by_suffix(headers, suffix_lower, exact_keys) -} - -fn snowball_meta_flag(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> bool { - snowball_meta_value(headers, exact_keys, suffix_lower).is_some_and(|value| value.eq_ignore_ascii_case("true")) -} - -pub fn normalize_snowball_prefix(prefix: &str) -> Option { - let normalized = prefix.trim().trim_matches('/'); - if normalized.is_empty() { - return None; - } - - Some(normalized.to_string()) -} - -pub fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -> String { - let path = path.trim_matches('/'); - let mut key = match prefix { - Some(prefix) if !path.is_empty() => format!("{prefix}/{path}"), - Some(prefix) => prefix.to_string(), - None => path.to_string(), - }; - - if is_dir && !key.ends_with('/') { - key.push('/'); - } - - key -} - -pub fn resolve_put_object_extract_options(headers: &HeaderMap) -> PutObjectExtractOptions { - let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER) - .and_then(|value| normalize_snowball_prefix(&value)); - let ignore_dirs = snowball_meta_flag(headers, SNOWBALL_IGNORE_DIRS_HEADER_KEYS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER); - let ignore_errors = snowball_meta_flag(headers, SNOWBALL_IGNORE_ERRORS_HEADER_KEYS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER); - - PutObjectExtractOptions { - prefix, - ignore_dirs, - ignore_errors, - } -} - -pub fn map_extract_archive_error(err: impl std::fmt::Display) -> S3Error { - s3_error!(InvalidArgument, "Failed to process archive entry: {}", err) -} - -pub async fn apply_extract_entry_pax_extensions( - entry: &mut tokio_tar::Entry>, - metadata: &mut HashMap, - opts: &mut ObjectOptions, -) -> s3s::S3Result<()> -where - R: AsyncRead + Send + Unpin + 'static, -{ - let Some(extensions) = entry.pax_extensions().await.map_err(map_extract_archive_error)? else { - return Ok(()); - }; - - for ext in extensions { - let ext = ext.map_err(map_extract_archive_error)?; - let key = ext.key().map_err(map_extract_archive_error)?; - let value = ext.value().map_err(map_extract_archive_error)?; - - if let Some(meta_key) = key.strip_prefix("minio.metadata.") { - let meta_key = meta_key.strip_prefix("x-amz-meta-").unwrap_or(meta_key); - if !meta_key.is_empty() { - metadata.insert(meta_key.to_string(), value.to_string()); - } - continue; - } - - if key == "minio.versionId" && !value.is_empty() { - opts.version_id = Some(value.to_string()); - } - } - - Ok(()) -} - -pub fn is_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool { - input - .server_side_encryption - .as_ref() - .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) - || input.ssekms_key_id.is_some() - || headers - .get(AMZ_SERVER_SIDE_ENCRYPTION) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.trim().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) - || headers.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID) -} - -pub fn is_post_object_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool { - is_sse_kms_requested(input, headers) -} - -#[cfg(test)] -mod tests { - use super::*; - use bytes::Bytes; - use http::{HeaderMap, HeaderName, HeaderValue}; - use std::io::Cursor; - use tokio::io::AsyncReadExt; - - #[test] - fn should_use_zero_copy_accepts_large_unencrypted_binary_payload() { - let mut headers = HeaderMap::new(); - headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); - assert!(should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_small_payloads() { - assert!(!should_use_zero_copy(512 * 1024, &HeaderMap::new())); - } - - #[test] - fn resolve_put_body_size_uses_content_length_when_present() { - assert_eq!(resolve_put_body_size(Some(123), &HeaderMap::new()).unwrap(), 123); - } - - #[test] - fn resolve_put_body_size_uses_decoded_content_length_header() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_DECODED_CONTENT_LENGTH, "456".parse().unwrap()); - assert_eq!(resolve_put_body_size(None, &headers).unwrap(), 456); - } - - #[test] - fn should_use_zero_copy_rejects_encrypted_payloads() { - let mut headers = HeaderMap::new(); - headers.insert("x-amz-server-side-encryption", HeaderValue::from_static("AES256")); - assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_compressible_payloads() { - let mut headers = HeaderMap::new(); - headers.insert("content-type", HeaderValue::from_static("application/json")); - assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_boundary_at_1mb() { - let headers = HeaderMap::new(); - - assert!(!should_use_zero_copy(1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_small_objects() { - let headers = HeaderMap::new(); - - assert!(!should_use_zero_copy(1024 * 1024 - 1, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_one_megabyte() { - let headers = HeaderMap::new(); - - assert!(!should_use_zero_copy(1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_encrypted_requests() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("AES256")); - - assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_encrypted_requests_with_sse_customer_algorithm() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, HeaderValue::from_static("AES256")); - - assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_encrypted_requests_with_kms_key_id() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, HeaderValue::from_static("test-kms-key-id")); - - assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_compressible_content_types() { - let mut headers = HeaderMap::new(); - headers.insert( - rustfs_utils::http::CONTENT_TYPE, - HeaderValue::from_static("application/json; charset=utf-8"), - ); - - assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_allows_large_unencrypted_binary_objects() { - let mut headers = HeaderMap::new(); - headers.insert(rustfs_utils::http::CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); - - assert!(should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn plan_put_object_ingress_preserves_buffer_size_and_fast_path_decision() { - let mut headers = HeaderMap::new(); - headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); - - let plan = plan_put_object_ingress(2 * 1024 * 1024, &headers, 256 * 1024); - - assert_eq!(plan.kind, PutObjectIngressKind::ReducedCopyCandidate); - assert_eq!(plan.compat.kind, PutObjectCompatIngressKind::BufferedStreamCompat); - assert_eq!(plan.compat.buffer_size, 256 * 1024); - assert!(plan.enable_zero_copy); - } - - #[test] - fn plan_put_object_body_disables_compression_for_small_payloads() { - let plan = plan_put_object_body(1024, &HeaderMap::new(), "small.bin", 64 * 1024); - - assert_eq!(plan.ingress.kind, PutObjectIngressKind::LegacyCompat); - assert_eq!(plan.ingress.compat.buffer_size, 64 * 1024); - assert!(!plan.ingress.enable_zero_copy); - assert_eq!(plan.kind, PutObjectBodyKind::Plain(PutObjectPlainBodyKind::LegacyCompat)); - assert!(!plan.should_compress()); - } - - #[test] - fn plan_put_object_body_marks_large_plain_payload_as_reduced_copy_candidate() { - let mut headers = HeaderMap::new(); - headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); - - let plan = plan_put_object_body(2 * 1024 * 1024, &headers, "large.bin", 256 * 1024); - - assert_eq!(plan.kind, PutObjectBodyKind::Plain(PutObjectPlainBodyKind::ReducedCopyCandidate)); - assert_eq!(plan.plain_body_kind(), Some(PutObjectPlainBodyKind::ReducedCopyCandidate)); - assert!(!plan.should_compress()); - } - - #[test] - fn plan_put_object_body_with_transforms_allows_encrypted_large_binary_payloads() { - let mut headers = HeaderMap::new(); - headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); - headers.insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("AES256")); - - let plan = plan_put_object_body_with_transforms(2 * 1024 * 1024, &headers, "large.bin", 256 * 1024, true); - - assert_eq!(plan.kind, PutObjectBodyKind::Plain(PutObjectPlainBodyKind::ReducedCopyCandidate)); - assert_eq!(plan.plain_body_kind(), Some(PutObjectPlainBodyKind::ReducedCopyCandidate)); - assert!(plan.ingress.enable_zero_copy); - } - - #[test] - fn build_put_object_ingress_source_preserves_reduced_copy_candidate_for_compressed_body() { - let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"compressed"))]); - let plan = PutObjectBodyPlan { - ingress: PutObjectIngressPlan { - kind: PutObjectIngressKind::ReducedCopyCandidate, - compat: PutObjectCompatIngressPlan { - kind: PutObjectCompatIngressKind::BufferedStreamCompat, - buffer_size: 256 * 1024, - }, - enable_zero_copy: true, - }, - kind: PutObjectBodyKind::Compressed, - }; - - let source = build_put_object_ingress_source(stream, plan); - assert!(matches!(source, PutObjectIngressSource::ReducedCopyCandidate(_))); - } - - #[test] - fn put_object_body_plan_reports_compressed_kind() { - let plan = PutObjectBodyPlan { - ingress: PutObjectIngressPlan { - kind: PutObjectIngressKind::LegacyCompat, - compat: PutObjectCompatIngressPlan { - kind: PutObjectCompatIngressKind::BufferedStreamCompat, - buffer_size: 256 * 1024, - }, - enable_zero_copy: false, - }, - kind: PutObjectBodyKind::Compressed, - }; - - assert_eq!(plan.plain_body_kind(), None); - assert!(plan.should_compress()); - } - - #[test] - fn resolve_put_effective_copy_mode_marks_transformed_paths() { - assert_eq!(resolve_put_effective_copy_mode(false, false), rustfs_io_metrics::CopyMode::SingleCopy); - assert_eq!(resolve_put_effective_copy_mode(true, false), rustfs_io_metrics::CopyMode::Transformed); - assert_eq!(resolve_put_effective_copy_mode(false, true), rustfs_io_metrics::CopyMode::Transformed); - } - - #[test] - fn put_object_transform_stage_tracks_transform_shape() { - let mut stage = PutObjectTransformStage::default(); - assert_eq!(stage.effective_copy_mode(), rustfs_io_metrics::CopyMode::SingleCopy); - assert_eq!(stage.metric_kind(), None); - - stage.mark_compression(); - assert!(stage.compression_applied()); - assert_eq!(stage.effective_copy_mode(), rustfs_io_metrics::CopyMode::Transformed); - assert_eq!(stage.metric_kind(), Some("compression")); - - stage.mark_encryption(); - assert!(stage.encryption_applied()); - assert_eq!(stage.metric_kind(), Some("compression_encryption")); - } - - #[test] - fn resolve_put_transform_metric_kind_reports_transform_shape() { - assert_eq!(resolve_put_transform_metric_kind(false, false), None); - assert_eq!(resolve_put_transform_metric_kind(true, false), Some("compression")); - assert_eq!(resolve_put_transform_metric_kind(false, true), Some("encryption")); - assert_eq!(resolve_put_transform_metric_kind(true, true), Some("compression_encryption")); - } - - #[test] - fn resolve_put_transformed_fallback_reason_isolated_from_plain_path() { - assert_eq!( - resolve_put_transformed_fallback_reason(PutObjectIngressKind::LegacyCompat, true, false), - Some(rustfs_io_metrics::FallbackReason::TransformCompressionLegacy) - ); - assert_eq!( - resolve_put_transformed_fallback_reason(PutObjectIngressKind::LegacyCompat, false, true), - Some(rustfs_io_metrics::FallbackReason::TransformEncryptionLegacy) - ); - assert_eq!( - resolve_put_transformed_fallback_reason(PutObjectIngressKind::LegacyCompat, true, true), - Some(rustfs_io_metrics::FallbackReason::TransformCompressionEncryptionLegacy) - ); - assert_eq!( - resolve_put_transformed_fallback_reason(PutObjectIngressKind::ReducedCopyCandidate, true, true), - None - ); - } - - #[test] - fn is_put_object_extract_requested_accepts_meta_header() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); - assert!(is_put_object_extract_requested(&headers)); - } - - #[test] - fn is_put_object_extract_requested_accepts_compat_header_case_insensitive() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_EXTRACT_COMPAT, HeaderValue::from_static(" TRUE ")); - assert!(is_put_object_extract_requested(&headers)); - } - - #[test] - fn is_put_object_extract_requested_rejects_missing_or_false_value() { - let mut headers = HeaderMap::new(); - assert!(!is_put_object_extract_requested(&headers)); - headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("false")); - assert!(!is_put_object_extract_requested(&headers)); - } - - #[test] - fn normalize_snowball_prefix_trims_slashes_and_whitespace() { - assert_eq!(normalize_snowball_prefix(" /batch/incoming/ "), Some("batch/incoming".to_string())); - assert_eq!(normalize_snowball_prefix("///"), None); - } - - #[test] - fn normalize_extract_entry_key_applies_prefix_and_directory_suffix() { - assert_eq!( - normalize_extract_entry_key("nested/path.txt", Some("imports"), false), - "imports/nested/path.txt" - ); - assert_eq!(normalize_extract_entry_key("nested/dir/", Some("imports"), true), "imports/nested/dir/"); - assert_eq!(normalize_extract_entry_key("top-level", None, false), "top-level"); - } - - #[test] - fn resolve_put_object_extract_options_defaults_when_headers_missing() { - let headers = HeaderMap::new(); - let options = resolve_put_object_extract_options(&headers); - assert_eq!( - options, - PutObjectExtractOptions { - prefix: None, - ignore_dirs: false, - ignore_errors: false - } - ); - } - - #[test] - fn resolve_put_object_extract_options_accepts_internal_headers() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_PREFIX_INTERNAL, HeaderValue::from_static("/internal/prefix/")); - headers.insert(AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, HeaderValue::from_static("true")); - headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, HeaderValue::from_static("TRUE")); - - let options = resolve_put_object_extract_options(&headers); - assert_eq!(options.prefix.as_deref(), Some("internal/prefix")); - assert!(options.ignore_dirs); - assert!(options.ignore_errors); - } - - #[test] - fn resolve_put_object_extract_options_accepts_standard_headers() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static(" /standard/prefix/ ")); - headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static(" true ")); - headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("TRUE")); - - let options = resolve_put_object_extract_options(&headers); - assert_eq!(options.prefix.as_deref(), Some("standard/prefix")); - assert!(options.ignore_dirs); - assert!(options.ignore_errors); - } - - #[test] - fn resolve_put_object_extract_options_accepts_suffix_compatible_headers() { - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("x-amz-meta-acme-snowball-prefix"), - HeaderValue::from_static(" /partner/import "), - ); - headers.insert( - HeaderName::from_static("x-amz-meta-acme-snowball-ignore-dirs"), - HeaderValue::from_static(" true "), - ); - headers.insert( - HeaderName::from_static("x-amz-meta-acme-snowball-ignore-errors"), - HeaderValue::from_static("TRUE"), - ); - - let options = resolve_put_object_extract_options(&headers); - assert_eq!(options.prefix.as_deref(), Some("partner/import")); - assert!(options.ignore_dirs); - assert!(options.ignore_errors); - } - - #[test] - fn resolve_put_object_extract_options_prefers_exact_headers_over_suffix_fallback() { - let mut headers = HeaderMap::new(); - headers.insert("x-amz-meta-acme-snowball-prefix", HeaderValue::from_static("/fallback/prefix/")); - headers.insert(AMZ_RUSTFS_SNOWBALL_PREFIX, HeaderValue::from_static("/internal/prefix/")); - headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static("/standard/prefix/")); - headers.insert(AMZ_MINIO_SNOWBALL_PREFIX, HeaderValue::from_static("/minio/prefix/")); - - let options = resolve_put_object_extract_options(&headers); - assert_eq!(options.prefix.as_deref(), Some("minio/prefix")); - } - - #[test] - fn resolve_put_object_extract_options_exact_flags_override_suffix_fallback() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static("false")); - headers.insert("x-amz-meta-acme-snowball-ignore-dirs", HeaderValue::from_static("true")); - headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("false")); - headers.insert("x-amz-meta-acme-snowball-ignore-errors", HeaderValue::from_static("true")); - - let options = resolve_put_object_extract_options(&headers); - assert!(!options.ignore_dirs); - assert!(!options.ignore_errors); - } - - #[tokio::test] - async fn build_put_object_compat_ingress_reads_buffered_stream() { - let stream = futures_util::stream::iter([ - Ok::(Bytes::from_static(b"abc")), - Ok::(Bytes::from_static(b"def")), - ]); - let plan = PutObjectCompatIngressPlan { - kind: PutObjectCompatIngressKind::BufferedStreamCompat, - buffer_size: 8, - }; - - let mut reader = build_put_object_compat_ingress(stream, plan); - let mut buf = Vec::new(); - reader.read_to_end(&mut buf).await.unwrap(); - - assert_eq!(buf, b"abcdef"); - } - - #[tokio::test] - async fn build_put_object_compat_ingress_maps_stream_errors() { - let stream = futures_util::stream::iter([Err::("boom")]); - let plan = PutObjectCompatIngressPlan { - kind: PutObjectCompatIngressKind::BufferedStreamCompat, - buffer_size: 8, - }; - - let mut reader = build_put_object_compat_ingress(stream, plan); - let err = reader.read_to_end(&mut Vec::new()).await.unwrap_err(); - - assert_eq!(err.kind(), std::io::ErrorKind::Other); - assert!(err.to_string().contains("boom")); - } - - #[tokio::test] - async fn build_put_object_compat_reader_wraps_ingress_as_reader() { - let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"reader"))]); - let plan = PutObjectCompatIngressPlan { - kind: PutObjectCompatIngressKind::BufferedStreamCompat, - buffer_size: 16, - }; - - let mut reader = build_put_object_compat_reader(stream, plan); - let mut buf = Vec::new(); - reader.read_to_end(&mut buf).await.unwrap(); - - assert_eq!(buf, b"reader"); - } - - #[tokio::test] - async fn build_put_object_ingress_source_keeps_plain_candidate_as_reduced_copy_source() { - let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"candidate"))]); - let mut headers = HeaderMap::new(); - headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); - let plan = plan_put_object_body(2 * 1024 * 1024, &headers, "large.bin", 16); - - let source = build_put_object_ingress_source(stream, plan); - let candidate = match source { - PutObjectIngressSource::ReducedCopyCandidate(candidate) => candidate, - PutObjectIngressSource::LegacyCompat(_) => panic!("expected reduced-copy candidate"), - }; - - assert_eq!(candidate.compat_plan().kind, PutObjectCompatIngressKind::BufferedStreamCompat); - assert_eq!(candidate.compat_plan().buffer_size, 16); - - let mut reader = candidate.into_compat_reader(); - let mut buf = Vec::new(); - reader.read_to_end(&mut buf).await.unwrap(); - - assert_eq!(buf, b"candidate"); - } - - #[tokio::test] - async fn build_put_object_ingress_source_routes_compressed_body_to_legacy_compat_reader() { - let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"compressed"))]); - let mut headers = HeaderMap::new(); - headers.insert("content-type", HeaderValue::from_static("application/json")); - let plan = plan_put_object_body(2 * 1024 * 1024, &headers, "large.json", 32); - - let source = build_put_object_ingress_source(stream, plan); - let mut reader = match source { - PutObjectIngressSource::LegacyCompat(reader) => reader, - PutObjectIngressSource::ReducedCopyCandidate(_) => panic!("expected legacy compat reader"), - }; - - let mut buf = Vec::new(); - reader.read_to_end(&mut buf).await.unwrap(); - - assert_eq!(buf, b"compressed"); - } - - #[tokio::test] - async fn build_put_object_reduced_copy_reader_reads_across_multiple_chunks() { - let stream = futures_util::stream::iter([ - Ok::(Bytes::from_static(b"ab")), - Ok::(Bytes::from_static(b"cd")), - Ok::(Bytes::from_static(b"ef")), - ]); - - let mut reader = build_put_object_reduced_copy_reader(PutObjectReducedCopyIngress::from_stream( - stream, - PutObjectCompatIngressPlan { - kind: PutObjectCompatIngressKind::BufferedStreamCompat, - buffer_size: 16, - }, - )); - let mut buf = Vec::new(); - reader.read_to_end(&mut buf).await.unwrap(); - - assert_eq!(buf, b"abcdef"); - } - - #[tokio::test] - async fn build_put_object_reduced_copy_reader_defers_stream_error_until_next_read() { - let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"ab")), Err::("boom")]); - - let mut reader = build_put_object_reduced_copy_reader(PutObjectReducedCopyIngress::from_stream( - stream, - PutObjectCompatIngressPlan { - kind: PutObjectCompatIngressKind::BufferedStreamCompat, - buffer_size: 16, - }, - )); - - let mut buf = [0_u8; 8]; - let n = reader.read(&mut buf).await.unwrap(); - assert_eq!(n, 2); - assert_eq!(&buf[..n], b"ab"); - - let err = reader.read(&mut buf).await.unwrap_err(); - assert_eq!(err.kind(), std::io::ErrorKind::Other); - assert!(err.to_string().contains("boom")); - } - - #[tokio::test] - async fn build_put_object_plain_hash_stage_preserves_legacy_compat_boundary() { - let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"legacy"))]); - let plan = plan_put_object_body(1024, &HeaderMap::new(), "small.bin", 32); - - let stage = build_put_object_plain_hash_stage( - build_put_object_ingress_source(stream, plan), - PutObjectLegacyHashValues::default(), - PutObjectLegacyHashStagePlan { - size: 6, - actual_size: 6, - apply_s3_checksum: false, - ignore_s3_checksum_value: false, - }, - &HeaderMap::new(), - None, - ) - .unwrap(); - - assert_eq!(stage.ingress_kind, PutObjectIngressKind::LegacyCompat); - - let mut reader = stage.reader; - let mut buf = Vec::new(); - reader.read_to_end(&mut buf).await.unwrap(); - - assert_eq!(buf, b"legacy"); - } - - #[test] - fn put_object_legacy_hash_values_clear_for_transformed_body_resets_hashes() { - let mut hash_values = PutObjectLegacyHashValues { - md5hex: Some("md5".to_string()), - sha256hex: Some("sha256".to_string()), - }; - - hash_values.clear_for_transformed_body(); - - assert_eq!(hash_values, PutObjectLegacyHashValues::default()); - } - - #[test] - fn build_put_object_legacy_hash_stage_preserves_legacy_compat_boundary() { - let reader: Box = Box::new(WarpReader::new(Cursor::new(Vec::from(&b"abc"[..])))); - let stage = build_put_object_legacy_hash_stage( - reader, - PutObjectLegacyHashValues::default(), - PutObjectLegacyHashStagePlan { - size: 3, - actual_size: 3, - apply_s3_checksum: false, - ignore_s3_checksum_value: false, - }, - &HeaderMap::new(), - None, - ) - .unwrap(); - - assert_eq!(stage.reader.size(), 3); - assert_eq!(stage.reader.actual_size(), 3); - assert!(stage.want_checksum.is_none()); - } -} diff --git a/crates/rio/src/hash_reader.rs b/crates/rio/src/hash_reader.rs index 98fb70037..aee0a50d6 100644 --- a/crates/rio/src/hash_reader.rs +++ b/crates/rio/src/hash_reader.rs @@ -408,23 +408,6 @@ impl HashReader { Ok(()) } - pub fn enable_auto_checksum(&mut self, checksum_type: ChecksumType) -> Result<(), std::io::Error> { - if !checksum_type.is_set() { - return Ok(()); - } - - let Some(hasher) = checksum_type.hasher() else { - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid checksum type")); - }; - - self.content_hash = Some(Checksum { - checksum_type, - ..Default::default() - }); - self.content_hasher = Some(hasher); - Ok(()) - } - pub fn checksum(&self) -> Option { if self .content_hash @@ -564,9 +547,7 @@ impl AsyncRead for HashReader { } // check content hasher - if let (Some(hasher), Some(expected_content_hash)) = - (this.content_hasher.as_mut(), this.content_hash.as_mut()) - { + if let (Some(hasher), Some(expected_content_hash)) = (this.content_hasher, this.content_hash) { if expected_content_hash.checksum_type.trailing() && let Some(trailer) = this.trailer_s3s.as_ref() && let Some(Some(checksum_str)) = trailer.read(|headers| { @@ -588,11 +569,7 @@ impl AsyncRead for HashReader { let content_hash = hasher.finalize(); - if expected_content_hash.encoded.is_empty() { - expected_content_hash.raw = content_hash.clone(); - expected_content_hash.encoded = general_purpose::STANDARD.encode(&content_hash); - *this.content_hash = Some(expected_content_hash.clone()); - } else if content_hash != expected_content_hash.raw { + if content_hash != expected_content_hash.raw { let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower); let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower); error!( diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 2242d4598..df1042e3a 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -90,7 +90,6 @@ rustfs-utils = { workspace = true, features = ["full"] } rustfs-zip = { workspace = true } rustfs-io-core = { workspace = true } rustfs-io-metrics = { workspace = true } -rustfs-object-io = { workspace = true } rustfs-object-capacity = { workspace = true } rustfs-concurrency = { workspace = true } rustfs-scanner = { workspace = true } diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 788d1141f..a4219825b 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -48,8 +48,7 @@ use rustfs_ecstore::set_disk::is_valid_storage_class; use rustfs_ecstore::store_api::{CompletePart, HTTPRangeSpec, MultipartUploadResult, ObjectIO, ObjectOptions, PutObjReader}; use rustfs_ecstore::store_api::{MultipartOperations, ObjectOperations}; use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; -use rustfs_object_io::put::PutObjectChecksums; -use rustfs_rio::{CompressReader, HashReader, Reader, WarpReader}; +use rustfs_rio::{CompressReader, HashReader}; use rustfs_s3_common::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; @@ -651,13 +650,6 @@ impl DefaultMultipartUsecase { .map_err(ApiError::from)?; let mut size = size.ok_or_else(|| s3_error!(UnexpectedContent))?; - let mut requested_checksum_type = rustfs_rio::ChecksumType::from_header(&req.headers); - if !requested_checksum_type.is_set() - && let Some(checksum_algo) = fi.user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM) - && let Some(checksum_type) = fi.user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE) - { - requested_checksum_type = rustfs_rio::ChecksumType::from_string_with_obj_type(checksum_algo, checksum_type); - } // Apply adaptive buffer sizing based on part size for optimal streaming performance. // Uses workload profile configuration (enabled by default) to select appropriate buffer size. @@ -670,8 +662,6 @@ impl DefaultMultipartUsecase { let is_compressible = rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); - let mut reader: Box = Box::new(WarpReader::new(body)); - let actual_size = size; let mut md5hex = if let Some(base64_md5) = input.content_md5 { @@ -685,32 +675,31 @@ impl DefaultMultipartUsecase { let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); - if is_compressible { - let mut hrd = - HashReader::new(reader, size, actual_size, md5hex.take(), sha256hex.take(), false).map_err(ApiError::from)?; + let mut reader = if is_compressible { + let mut hrd = HashReader::from_stream(body, size, actual_size, md5hex.take(), sha256hex.take(), false) + .map_err(ApiError::from)?; if let Err(err) = hrd.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) { return Err(ApiError::from(err).into()); } - if requested_checksum_type.is_set() && hrd.checksum().is_none() { - hrd.enable_auto_checksum(requested_checksum_type).map_err(ApiError::from)?; - } - let compress_reader = CompressReader::new(hrd, CompressionAlgorithm::default()); - reader = Box::new(compress_reader); size = HashReader::SIZE_PRESERVE_LAYER; - md5hex = None; - sha256hex = None; - } - - let mut reader = HashReader::new(reader, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; + HashReader::from_reader( + CompressReader::new(hrd, CompressionAlgorithm::default()), + size, + actual_size, + None, + None, + false, + ) + .map_err(ApiError::from)? + } else { + HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? + }; if let Err(err) = reader.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), size < 0) { return Err(ApiError::from(err).into()); } - if requested_checksum_type.is_set() && reader.checksum().is_none() { - reader.enable_auto_checksum(requested_checksum_type).map_err(ApiError::from)?; - } let has_ssec = sse_customer_algorithm.is_some(); // When SSE-C headers are present, skip managed-encryption metadata to avoid @@ -760,8 +749,9 @@ impl DefaultMultipartUsecase { let requested_kms_key_id = material.kms_key_id.clone(); let encrypted_reader = material.wrap_reader(reader); - reader = HashReader::new(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; + reader = + HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + .map_err(ApiError::from)?; fi.user_defined.extend(material.metadata); @@ -777,13 +767,11 @@ impl DefaultMultipartUsecase { .await .map_err(ApiError::from)?; - let mut checksums = PutObjectChecksums { - crc32: input.checksum_crc32, - crc32c: input.checksum_crc32c, - sha1: input.checksum_sha1, - sha256: input.checksum_sha256, - crc64nvme: input.checksum_crc64nvme, - }; + let mut checksum_crc32 = input.checksum_crc32; + let mut checksum_crc32c = input.checksum_crc32c; + let mut checksum_sha1 = input.checksum_sha1; + let mut checksum_sha256 = input.checksum_sha256; + let mut checksum_crc64nvme = input.checksum_crc64nvme; if let Some(alg) = &input.checksum_algorithm && let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| { @@ -803,26 +791,25 @@ impl DefaultMultipartUsecase { }) { match alg.as_str() { - ChecksumAlgorithm::CRC32 => checksums.crc32 = checksum_str, - ChecksumAlgorithm::CRC32C => checksums.crc32c = checksum_str, - ChecksumAlgorithm::SHA1 => checksums.sha1 = checksum_str, - ChecksumAlgorithm::SHA256 => checksums.sha256 = checksum_str, - ChecksumAlgorithm::CRC64NVME => checksums.crc64nvme = checksum_str, + ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str, + ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str, + ChecksumAlgorithm::SHA1 => checksum_sha1 = checksum_str, + ChecksumAlgorithm::SHA256 => checksum_sha256 = checksum_str, + ChecksumAlgorithm::CRC64NVME => checksum_crc64nvme = checksum_str, _ => (), } } - checksums.merge_from_map(&reader.as_hash_reader().content_crc()); let output = UploadPartOutput { server_side_encryption: requested_sse, ssekms_key_id: requested_kms_key_id, sse_customer_algorithm, sse_customer_key_md5, - checksum_crc32: checksums.crc32, - checksum_crc32c: checksums.crc32c, - checksum_sha1: checksums.sha1, - checksum_sha256: checksums.sha256, - checksum_crc64nvme: checksums.crc64nvme, + checksum_crc32, + checksum_crc32c, + checksum_sha1, + checksum_sha256, + checksum_crc64nvme, e_tag: info.etag.map(|etag| to_s3s_etag(&etag)), ..Default::default() }; @@ -1013,8 +1000,6 @@ impl DefaultMultipartUsecase { let is_compressible = rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); - let mut reader: Box = Box::new(WarpReader::new(src_stream)); - let src_decryption_request = DecryptionRequest { bucket: &src_bucket, key: &src_key, @@ -1026,23 +1011,74 @@ impl DefaultMultipartUsecase { etag: src_info.etag.as_deref(), }; - if let Some(material) = sse_decryption(src_decryption_request).await? { - reader = material.wrap_single_reader(reader); - if let Some(original) = material.original_size { - src_info.actual_size = original; - } - } - let actual_size = length; let mut size = length; - if is_compressible { - let hrd = HashReader::new(reader, size, actual_size, None, None, false).map_err(ApiError::from)?; - reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default())); - size = HashReader::SIZE_PRESERVE_LAYER; - } + let mut reader = match sse_decryption(src_decryption_request).await? { + Some(material) => { + if let Some(original) = material.original_size { + src_info.actual_size = original; + } - let mut reader = HashReader::new(reader, size, actual_size, None, None, false).map_err(ApiError::from)?; + if material.is_multipart { + let (decrypted_stream, plaintext_size) = + material.wrap_reader(src_stream, size).await.map_err(ApiError::from)?; + size = plaintext_size; + + if is_compressible { + let hrd = HashReader::from_reader(decrypted_stream, size, actual_size, None, None, false) + .map_err(ApiError::from)?; + size = HashReader::SIZE_PRESERVE_LAYER; + HashReader::from_reader( + CompressReader::new(hrd, CompressionAlgorithm::default()), + size, + actual_size, + None, + None, + false, + ) + .map_err(ApiError::from)? + } else { + HashReader::from_reader(decrypted_stream, size, actual_size, None, None, false).map_err(ApiError::from)? + } + } else if is_compressible { + let hrd = + HashReader::from_stream(material.wrap_single_reader(src_stream), size, actual_size, None, None, false) + .map_err(ApiError::from)?; + size = HashReader::SIZE_PRESERVE_LAYER; + HashReader::from_reader( + CompressReader::new(hrd, CompressionAlgorithm::default()), + size, + actual_size, + None, + None, + false, + ) + .map_err(ApiError::from)? + } else { + HashReader::from_stream(material.wrap_single_reader(src_stream), size, actual_size, None, None, false) + .map_err(ApiError::from)? + } + } + None => { + if is_compressible { + let hrd = + HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)?; + size = HashReader::SIZE_PRESERVE_LAYER; + HashReader::from_reader( + CompressReader::new(hrd, CompressionAlgorithm::default()), + size, + actual_size, + None, + None, + false, + ) + .map_err(ApiError::from)? + } else { + HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)? + } + } + }; let server_side_encryption = mp_info .user_defined @@ -1083,8 +1119,9 @@ impl DefaultMultipartUsecase { let requested_kms_key_id = material.kms_key_id.clone(); let encrypted_reader = material.wrap_reader(reader); - reader = HashReader::new(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; + reader = + HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + .map_err(ApiError::from)?; mp_info.user_defined.extend(material.metadata); diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 4ccc40dae..ea2ad49e2 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -14,26 +14,21 @@ //! Object application use-case contracts. -mod get_object_flow; -mod get_object_zero_copy; -mod put_object_extract; -mod put_object_flow; -use self::get_object_flow::GetObjectBootstrap; - use crate::app::context::{AppContext, default_notify_interface, get_global_app_context}; -use crate::capacity::record_capacity_write; use crate::config::RustFSBufferConfig; use crate::error::ApiError; use crate::storage::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut}; -use crate::storage::concurrency::{GetObjectGuard, get_concurrency_manager}; +use crate::storage::concurrency::{ + ConcurrencyManager, GetObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, +}; use crate::storage::ecfs::*; use crate::storage::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children}; -use crate::storage::helper::{OperationHelper, spawn_background}; +use crate::storage::helper::{OperationHelper, spawn_background, spawn_background_with_context}; use crate::storage::options::{ copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, put_opts, - validate_archive_content_encoding, }; +use crate::storage::request_context::spawn_traced; use crate::storage::s3_api::multipart::parse_list_parts_params; use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig}; use crate::storage::*; @@ -41,10 +36,14 @@ use bytes::Bytes; use datafusion::arrow::{ csv::WriterBuilder as CsvWriterBuilder, json::WriterBuilder as JsonWriterBuilder, json::writer::JsonArray, }; -use futures::StreamExt; +use futures::{StreamExt, stream}; use http::{HeaderMap, HeaderValue, StatusCode}; use md5::Context as Md5Context; +use metrics::{counter, histogram}; use pin_project_lite::pin_project; +use rustfs_object_capacity::capacity_manager::get_capacity_manager; +// Performance metrics recording (with zero-copy-metrics integration) +use rustfs_concurrency::GetObjectQueueSnapshot; use rustfs_ecstore::bucket::quota::checker::QuotaChecker; use rustfs_ecstore::bucket::{ lifecycle::{ @@ -81,22 +80,26 @@ use rustfs_filemeta::{ }; use rustfs_io_metrics; use rustfs_notify::EventArgsBuilder; -use rustfs_object_io::put::{ - is_post_object_sse_kms_requested, is_put_object_extract_requested, is_sse_kms_requested, resolve_put_body_size, -}; use rustfs_policy::policy::action::{Action, S3Action}; -use rustfs_rio::{CompressReader, EtagReader, HashReader, Reader, WarpReader}; +use rustfs_rio::{CompressReader, DynReader, HashReader, wrap_reader}; use rustfs_s3_common::S3Operation; -use rustfs_s3select_api::query::{Context, Query}; +use rustfs_s3select_api::{ + object_store::bytes_stream, + query::{Context, Query}, +}; use rustfs_s3select_query::get_global_db; use rustfs_targets::EventName; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, headers::{ + AMZ_DECODED_CONTENT_LENGTH, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, AMZ_MINIO_SNOWBALL_PREFIX, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, - AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, + AMZ_RESTORE_REQUEST_DATE, AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, AMZ_RUSTFS_SNOWBALL_PREFIX, + AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, + AMZ_SNOWBALL_EXTRACT, AMZ_SNOWBALL_IGNORE_DIRS, AMZ_SNOWBALL_IGNORE_ERRORS, AMZ_SNOWBALL_PREFIX, AMZ_STORAGE_CLASS, + AMZ_TAG_COUNT, }, insert_str, remove_str, }; @@ -121,7 +124,7 @@ use tokio::sync::RwLock; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tokio_tar::Archive; -use tokio_util::io::StreamReader; +use tokio_util::io::{ReaderStream, StreamReader}; use tracing::{debug, error, info, instrument, warn}; use uuid::Uuid; @@ -130,34 +133,6 @@ struct DeadlockRequestGuard { request_id: String, } -#[derive(Clone)] -pub(super) struct GetObjectRequestContext { - pub(super) bucket: String, - pub(super) key: String, - pub(super) part_number: Option, - pub(super) rs: Option, - pub(super) opts: ObjectOptions, - pub(super) headers: HeaderMap, - pub(super) sse_customer_key: Option, - pub(super) sse_customer_key_md5: Option, -} - -pub(super) type PutObjectChecksums = rustfs_object_io::put::PutObjectChecksums; - -#[derive(Clone)] -pub(super) struct PutObjectRequestContext { - pub(super) headers: HeaderMap, - pub(super) trailing_headers: Option, - pub(super) uri_query: Option, - pub(super) is_post_object: bool, - pub(super) method: hyper::Method, - pub(super) uri: hyper::Uri, - pub(super) extensions: http::Extensions, - pub(super) credentials: Option, - pub(super) region: Option, - pub(super) service: Option, -} - impl DeadlockRequestGuard { fn new(deadlock_detector: Arc, request_id: String) -> Self { Self { @@ -173,24 +148,63 @@ impl Drop for DeadlockRequestGuard { } } -async fn resolve_bucket_default_server_side_encryption(bucket: &str) -> (Option, Option) { - let Some((config, _timestamp)) = metadata_sys::get_sse_config(bucket).await.ok() else { - return (None, None); - }; - let Some(default_sse) = config - .rules - .first() - .and_then(|rule| rule.apply_server_side_encryption_by_default.as_ref()) - else { - return (None, None); - }; +struct GetObjectBootstrap { + timeout_config: TimeoutConfig, + wrapper: RequestTimeoutWrapper, + request_start: std::time::Instant, + request_guard: GetObjectGuard, + _deadlock_request_guard: DeadlockRequestGuard, + concurrent_requests: usize, +} - let server_side_encryption = Some(match default_sse.sse_algorithm.as_str() { - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - }); +struct GetObjectIoPlanning<'a> { + _disk_permit: tokio::sync::SemaphorePermit<'a>, + permit_wait_duration: Duration, + queue_status: concurrency::IoQueueStatus, + queue_utilization: f64, +} - (server_side_encryption, default_sse.kms_master_key_id.clone()) +struct GetObjectRequestContext { + bucket: String, + key: String, + version_id_for_event: String, + part_number: Option, + rs: Option, + opts: ObjectOptions, +} + +struct GetObjectReadSetup { + info: ObjectInfo, + event_info: ObjectInfo, + final_stream: DynReader, + rs: Option, + content_type: Option, + last_modified: Option, + response_content_length: i64, + content_range: Option, + server_side_encryption: Option, + sse_customer_algorithm: Option, + sse_customer_key_md5: Option, + ssekms_key_id: Option, + encryption_applied: bool, +} + +struct GetObjectPreparedRead<'a> { + io_planning: GetObjectIoPlanning<'a>, + read_setup: GetObjectReadSetup, +} + +struct GetObjectStrategyContext { + #[allow(dead_code)] + io_strategy: concurrency::IoStrategy, + optimal_buffer_size: usize, +} + +struct GetObjectOutputContext { + output: GetObjectOutput, + event_info: ObjectInfo, + response_content_length: i64, + optimal_buffer_size: usize, } async fn enqueue_transitioned_delete_cleanup(bucket: &str, object: &str, opts: &ObjectOptions, existing: Option<&ObjectInfo>) { @@ -277,6 +291,61 @@ impl AsyncRead for ExtractArchiveEtagReader { } } +/// Determine if zero-copy write should be used for this PutObject operation. +/// +/// Zero-copy is beneficial for large objects without encryption or compression. +/// +/// # Arguments +/// +/// * `size` - Object size in bytes +/// * `headers` - HTTP headers (to check for encryption/compression) +/// +/// # Returns +/// +/// `true` if zero-copy should be used, `false` otherwise +fn should_use_zero_copy(size: i64, headers: &HeaderMap) -> bool { + // Only use zero-copy for objects larger than 1MB + const ZERO_COPY_MIN_SIZE: i64 = 1024 * 1024; + + if size <= ZERO_COPY_MIN_SIZE { + return false; + } + + // Don't use zero-copy if encryption is requested + if headers.get(AMZ_SERVER_SIDE_ENCRYPTION).is_some() + || headers.get(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM).is_some() + || headers.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID).is_some() + { + return false; + } + + // Don't use zero-copy if compression is likely (compressible content types) + // The compression check happens later in the flow + if let Some(content_type) = headers.get(CONTENT_TYPE) + && let Ok(ct) = content_type.to_str() + { + // Skip zero-copy for easily compressible content types + // since compression will be applied + let compressible_types = [ + "text/plain", + "text/html", + "text/css", + "text/javascript", + "application/javascript", + "application/json", + "application/xml", + "text/xml", + ]; + for ct_type in compressible_types { + if ct.contains(ct_type) { + return false; + } + } + } + + true +} + #[cfg(test)] mod deadlock_request_guard_tests { use super::DeadlockRequestGuard; @@ -302,6 +371,65 @@ mod deadlock_request_guard_tests { assert_eq!(detector.tracked_count(), 0); } } + +async fn maybe_enqueue_transition_immediate(obj_info: &ObjectInfo, src: LcEventSrc) { + enqueue_transition_immediate(obj_info, src).await; +} + +/// Extract trailing-header checksum values, overriding the corresponding input fields. +fn apply_trailing_checksums( + algorithm: Option<&str>, + trailing_headers: &Option, + checksums: &mut PutObjectChecksums, +) { + let Some(alg) = algorithm else { return }; + let Some(checksum_str) = trailing_headers.as_ref().and_then(|trailer| { + let key = match alg { + ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(), + ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(), + ChecksumAlgorithm::SHA1 => rustfs_rio::ChecksumType::SHA1.key(), + ChecksumAlgorithm::SHA256 => rustfs_rio::ChecksumType::SHA256.key(), + ChecksumAlgorithm::CRC64NVME => rustfs_rio::ChecksumType::CRC64_NVME.key(), + _ => return None, + }; + trailer.read(|headers| { + headers + .get(key.unwrap_or_default()) + .and_then(|value| value.to_str().ok().map(|s| s.to_string())) + }) + }) else { + return; + }; + + match alg { + ChecksumAlgorithm::CRC32 => checksums.crc32 = checksum_str, + ChecksumAlgorithm::CRC32C => checksums.crc32c = checksum_str, + ChecksumAlgorithm::SHA1 => checksums.sha1 = checksum_str, + ChecksumAlgorithm::SHA256 => checksums.sha256 = checksum_str, + ChecksumAlgorithm::CRC64NVME => checksums.crc64nvme = checksum_str, + _ => (), + } +} + +#[derive(Default)] +struct GetObjectChecksums { + crc32: Option, + crc32c: Option, + sha1: Option, + sha256: Option, + crc64nvme: Option, + checksum_type: Option, +} + +#[derive(Default)] +struct PutObjectChecksums { + crc32: Option, + crc32c: Option, + sha1: Option, + sha256: Option, + crc64nvme: Option, +} + fn normalize_delete_objects_version_id(version_id: Option) -> Result<(Option, Option), String> { let version_id = version_id.map(|v| v.trim().to_string()).filter(|v| !v.is_empty()); match version_id { @@ -332,56 +460,146 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option) -> S3Result { - let GetObjectInput { - bucket, - key, - version_id, - part_number, - range, - .. - } = req.input.clone(); +const AMZ_SNOWBALL_EXTRACT_COMPAT: &str = "X-Amz-Snowball-Auto-Extract"; +#[cfg(test)] +const AMZ_SNOWBALL_PREFIX_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix"; +#[cfg(test)] +const AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Dirs"; +#[cfg(test)] +const AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Errors"; +const AMZ_META_PREFIX_LOWER: &str = "x-amz-meta-"; +const SNOWBALL_PREFIX_SUFFIX_LOWER: &str = "snowball-prefix"; +const SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER: &str = "snowball-ignore-dirs"; +const SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER: &str = "snowball-ignore-errors"; +const SNOWBALL_PREFIX_HEADER_KEYS: &[&str] = &[AMZ_MINIO_SNOWBALL_PREFIX, AMZ_SNOWBALL_PREFIX, AMZ_RUSTFS_SNOWBALL_PREFIX]; +const SNOWBALL_IGNORE_DIRS_HEADER_KEYS: &[&str] = &[ + AMZ_MINIO_SNOWBALL_IGNORE_DIRS, + AMZ_SNOWBALL_IGNORE_DIRS, + AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, +]; +const SNOWBALL_IGNORE_ERRORS_HEADER_KEYS: &[&str] = &[ + AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, + AMZ_SNOWBALL_IGNORE_ERRORS, + AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, +]; - validate_object_key(&key, "GET")?; +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct PutObjectExtractOptions { + prefix: Option, + ignore_dirs: bool, + ignore_errors: bool, +} - let part_number = part_number.map(|value| value as usize); - if let Some(part_number) = part_number - && part_number == 0 - { - return Err(s3_error!(InvalidArgument, "Invalid part number: part number must be greater than 0")); +fn header_value_is_true(headers: &HeaderMap, key: &str) -> bool { + headers + .get(key) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case("true")) +} + +fn is_put_object_extract_requested(headers: &HeaderMap) -> bool { + header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_COMPAT) +} + +fn trimmed_header_value(headers: &HeaderMap, key: &str) -> Option { + headers + .get(key) + .and_then(|value| value.to_str().ok()) + .map(|value| value.trim().to_string()) +} + +fn is_exact_snowball_meta_key(key: &str, exact_keys: &[&str]) -> bool { + exact_keys.iter().any(|exact_key| key.eq_ignore_ascii_case(exact_key)) +} + +fn snowball_meta_value_by_suffix(headers: &HeaderMap, suffix_lower: &str, exact_keys: &[&str]) -> Option { + for (name, value) in headers { + let key = name.as_str(); + if key.starts_with(AMZ_META_PREFIX_LOWER) + && key.ends_with(suffix_lower) + && !is_exact_snowball_meta_key(key, exact_keys) + && let Ok(parsed) = value.to_str() + { + return Some(parsed.trim().to_string()); + } } - let rs = range.map(|value| match value { - Range::Int { first, last } => HTTPRangeSpec { - is_suffix_length: false, - start: first as i64, - end: last.map_or(-1, |last| last as i64), - }, - Range::Suffix { length } => HTTPRangeSpec { - is_suffix_length: true, - start: length as i64, - end: -1, - }, - }); + None +} - if rs.is_some() && part_number.is_some() { - return Err(s3_error!(InvalidArgument, "range and part_number invalid")); +fn snowball_meta_value(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> Option { + for key in exact_keys { + if let Some(value) = trimmed_header_value(headers, key) { + return Some(value); + } } - let opts: ObjectOptions = get_opts(&bucket, &key, version_id, part_number, &req.headers) - .await - .map_err(ApiError::from)?; + snowball_meta_value_by_suffix(headers, suffix_lower, exact_keys) +} - Ok(GetObjectRequestContext { - bucket, - key, - part_number, - rs, - opts, - headers: req.headers.clone(), - sse_customer_key: req.input.sse_customer_key.clone(), - sse_customer_key_md5: req.input.sse_customer_key_md5.clone(), - }) +fn snowball_meta_flag(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> bool { + snowball_meta_value(headers, exact_keys, suffix_lower).is_some_and(|value| value.eq_ignore_ascii_case("true")) +} + +fn normalize_snowball_prefix(prefix: &str) -> Option { + let normalized = prefix.trim().trim_matches('/'); + if normalized.is_empty() { + return None; + } + + Some(normalized.to_string()) +} + +fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -> String { + let path = path.trim_matches('/'); + let mut key = match prefix { + Some(prefix) if !path.is_empty() => format!("{prefix}/{path}"), + Some(prefix) => prefix.to_string(), + None => path.to_string(), + }; + + if is_dir && !key.ends_with('/') { + key.push('/'); + } + + key +} + +fn map_extract_archive_error(err: impl std::fmt::Display) -> S3Error { + s3_error!(InvalidArgument, "Failed to process archive entry: {}", err) +} + +async fn apply_extract_entry_pax_extensions( + entry: &mut tokio_tar::Entry>, + metadata: &mut HashMap, + opts: &mut ObjectOptions, +) -> S3Result<()> +where + R: AsyncRead + Send + Unpin + 'static, +{ + let Some(extensions) = entry.pax_extensions().await.map_err(map_extract_archive_error)? else { + return Ok(()); + }; + + for ext in extensions { + let ext = ext.map_err(map_extract_archive_error)?; + let key = ext.key().map_err(map_extract_archive_error)?; + let value = ext.value().map_err(map_extract_archive_error)?; + + if let Some(meta_key) = key.strip_prefix("minio.metadata.") { + let meta_key = meta_key.strip_prefix("x-amz-meta-").unwrap_or(meta_key); + if !meta_key.is_empty() { + metadata.insert(meta_key.to_string(), value.to_string()); + } + continue; + } + + if key == "minio.versionId" && !value.is_empty() { + opts.version_id = Some(value.to_string()); + } + } + + Ok(()) } #[allow(clippy::too_many_arguments)] @@ -399,20 +617,6 @@ fn apply_put_request_metadata( tagging: Option, storage_class: Option, ) -> S3Result<()> { - let request_content_type = content_type.as_ref().map(ToString::to_string).or_else(|| { - headers - .get("content-type") - .and_then(|value| value.to_str().ok()) - .map(ToOwned::to_owned) - }); - let request_content_encoding = content_encoding.as_ref().map(ToString::to_string).or_else(|| { - headers - .get("content-encoding") - .and_then(|value| value.to_str().ok()) - .map(ToOwned::to_owned) - }); - validate_archive_content_encoding(object_name, request_content_type.as_deref(), request_content_encoding.as_deref())?; - if let Some(cache_control) = cache_control { metadata.insert("cache-control".to_string(), cache_control.to_string()); } @@ -428,7 +632,7 @@ fn apply_put_request_metadata( metadata.insert("content-language".to_string(), content_language.to_string()); } if let Some(content_type) = content_type { - metadata.insert(CONTENT_TYPE.to_string(), content_type.to_string()); + metadata.insert("content-type".to_string(), content_type.to_string()); } if let Some(expires) = expires { let mut formatted = Vec::new(); @@ -539,6 +743,36 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool { opts.version_id.is_none() && opts.versioned && !opts.version_suspended } +fn resolve_put_object_extract_options(headers: &HeaderMap) -> PutObjectExtractOptions { + let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER) + .and_then(|value| normalize_snowball_prefix(&value)); + let ignore_dirs = snowball_meta_flag(headers, SNOWBALL_IGNORE_DIRS_HEADER_KEYS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER); + let ignore_errors = snowball_meta_flag(headers, SNOWBALL_IGNORE_ERRORS_HEADER_KEYS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER); + + PutObjectExtractOptions { + prefix, + ignore_dirs, + ignore_errors, + } +} + +fn is_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool { + input + .server_side_encryption + .as_ref() + .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) + || input.ssekms_key_id.is_some() + || headers + .get(AMZ_SERVER_SIDE_ENCRYPTION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) + || headers.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID) +} + +fn is_post_object_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool { + is_sse_kms_requested(input, headers) +} + async fn resolve_put_object_expiration(bucket: &str, obj_info: &ObjectInfo) -> Option { let Ok((lifecycle_config, _)) = metadata_sys::get_lifecycle_config(bucket).await else { debug!("resolve_put_object_expiration: lifecycle config not found for bucket {bucket}"); @@ -605,26 +839,576 @@ impl DefaultObjectUsecase { } } + fn build_memory_blob(buf: Vec, response_content_length: i64, _optimal_buffer_size: usize) -> Option { + Some(StreamingBlob::wrap(bytes_stream( + stream::once(async move { Ok::(Bytes::from(buf)) }), + response_content_length as usize, + ))) + } + + fn build_reader_blob(reader: R, response_content_length: i64, optimal_buffer_size: usize) -> Option + where + R: AsyncRead + Send + Sync + 'static, + { + Some(StreamingBlob::wrap(bytes_stream( + ReaderStream::with_capacity(reader, optimal_buffer_size), + response_content_length as usize, + ))) + } + + fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result { + let timeout_config = TimeoutConfig::from_env(); + let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.to_string()); + let request_start = std::time::Instant::now(); + let request_guard = ConcurrencyManager::track_request(); + let concurrent_requests = GetObjectGuard::concurrent_requests(); + + let deadlock_detector = deadlock_detector::get_deadlock_detector(); + let request_id = wrapper.request_id().to_string(); + deadlock_detector.register_request(&request_id, format!("GetObject {bucket}/{key}")); + let deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector, request_id); + + if wrapper.is_timeout() { + warn!( + bucket = %bucket, + key = %key, + timeout_secs = timeout_config.get_object_timeout.as_secs(), + elapsed_ms = wrapper.elapsed().as_millis(), + "GetObject request timed out before processing" + ); + return Err(s3_error!(InternalError, "Request timeout before processing")); + } + + rustfs_io_metrics::record_get_object_request_start(concurrent_requests); + + debug!( + "GetObject request started with {} concurrent requests, timeout={:?}", + concurrent_requests, timeout_config.get_object_timeout + ); + + Ok(GetObjectBootstrap { + timeout_config, + wrapper, + request_start, + request_guard, + _deadlock_request_guard: deadlock_request_guard, + concurrent_requests, + }) + } + + async fn acquire_get_object_io_planning<'a>( + manager: &'a ConcurrencyManager, + wrapper: &RequestTimeoutWrapper, + timeout_config: &TimeoutConfig, + bucket: &str, + key: &str, + ) -> S3Result> { + let permit_wait_start = std::time::Instant::now(); + let disk_permit = manager + .acquire_disk_read_permit() + .await + .map_err(|_| s3_error!(InternalError, "disk read semaphore closed"))?; + let permit_wait_duration = permit_wait_start.elapsed(); + + if wrapper.is_timeout() { + warn!( + bucket = %bucket, + key = %key, + wait_ms = permit_wait_duration.as_millis(), + timeout_secs = timeout_config.get_object_timeout.as_secs(), + elapsed_ms = wrapper.elapsed().as_millis(), + "GetObject request timed out while waiting for disk permit" + ); + + rustfs_io_metrics::record_get_object_timeout(Some("disk_permit"), Some(wrapper.elapsed().as_secs_f64())); + return Err(s3_error!(InternalError, "Request timeout while waiting for disk permit")); + } + + let queue_status = manager.io_queue_status(); + let queue_snapshot = GetObjectQueueSnapshot::from_available_permits( + queue_status.total_permits, + queue_status.total_permits.saturating_sub(queue_status.permits_in_use), + ); + let queue_utilization = queue_snapshot.utilization_percent(); + + if queue_snapshot.is_congested(80.0) { + warn!( + bucket = %bucket, + key = %key, + queue_utilization = format!("{:.1}%", queue_utilization), + permits_in_use = queue_status.permits_in_use, + total_permits = queue_status.total_permits, + "I/O queue congestion detected" + ); + + rustfs_io_metrics::record_io_queue_congestion(); + } + + if wrapper.is_timeout() { + warn!( + bucket = %bucket, + key = %key, + timeout_secs = timeout_config.get_object_timeout.as_secs(), + elapsed_ms = wrapper.elapsed().as_millis(), + "GetObject request timed out before reading object" + ); + rustfs_io_metrics::record_get_object_timeout(Some("before_read"), Some(wrapper.elapsed().as_secs_f64())); + return Err(s3_error!(InternalError, "Request timeout before reading object")); + } + + Ok(GetObjectIoPlanning { + _disk_permit: disk_permit, + permit_wait_duration, + queue_status, + queue_utilization, + }) + } + + async fn prepare_get_object_request_context(req: &S3Request) -> S3Result { + let GetObjectInput { + bucket, + key, + version_id, + part_number, + range, + .. + } = req.input.clone(); + + validate_object_key(&key, "GET")?; + + let part_number = part_number.map(|v| v as usize); + + if let Some(part_num) = part_number + && part_num == 0 + { + return Err(s3_error!(InvalidArgument, "Invalid part number: part number must be greater than 0")); + } + + let rs = range.map(|v| match v { + Range::Int { first, last } => HTTPRangeSpec { + is_suffix_length: false, + start: first as i64, + end: if let Some(last) = last { last as i64 } else { -1 }, + }, + Range::Suffix { length } => HTTPRangeSpec { + is_suffix_length: true, + start: length as i64, + end: -1, + }, + }); + + if rs.is_some() && part_number.is_some() { + return Err(s3_error!(InvalidArgument, "range and part_number invalid")); + } + + let opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), part_number, &req.headers) + .await + .map_err(ApiError::from)?; + + Ok(GetObjectRequestContext { + version_id_for_event: version_id.unwrap_or_default(), + bucket, + key, + part_number, + rs, + opts, + }) + } + #[allow(clippy::too_many_arguments)] + async fn prepare_get_object_read_execution<'a>( + req: &S3Request, + manager: &'a ConcurrencyManager, + wrapper: &RequestTimeoutWrapper, + timeout_config: &TimeoutConfig, + bucket: &str, + key: &str, + rs: Option, + opts: &ObjectOptions, + part_number: Option, + ) -> S3Result> { + let h = HeaderMap::new(); + let io_planning = Self::acquire_get_object_io_planning(manager, wrapper, timeout_config, bucket, key).await?; + let store = get_validated_store(bucket).await?; + + let read_start = std::time::Instant::now(); + let read_setup = + Self::prepare_get_object_read(req, &store, manager, bucket, key, rs, h, opts, part_number, read_start).await?; + + Ok(GetObjectPreparedRead { io_planning, read_setup }) + } + + #[allow(clippy::too_many_arguments)] + async fn prepare_get_object_read( + req: &S3Request, + store: &rustfs_ecstore::store::ECStore, + manager: &ConcurrencyManager, + bucket: &str, + key: &str, + mut rs: Option, + h: HeaderMap, + opts: &ObjectOptions, + part_number: Option, + read_start: std::time::Instant, + ) -> S3Result { + let reader = store + .get_object_reader(bucket, key, rs.clone(), h, opts) + .await + .map_err(ApiError::from)?; + + let info = reader.object_info; + + use rustfs_io_metrics::{record_memory_copy_saved, record_zero_copy_read}; + let read_duration = read_start.elapsed(); + let estimated_saved = (info.size * 2) as usize; + record_zero_copy_read(info.size as usize, read_duration.as_secs_f64() * 1000.0); + record_memory_copy_saved(estimated_saved); + + manager.record_disk_operation(info.size as u64, read_duration, true).await; + + check_preconditions(&req.headers, &info)?; + + debug!(object_size = info.size, part_count = info.parts.len(), "GET object metadata snapshot"); + for part in &info.parts { + debug!( + part_number = part.number, + part_size = part.size, + part_actual_size = part.actual_size, + "GET object part details" + ); + } + + let event_info = info.clone(); + let content_type = if let Some(content_type) = &info.content_type { + match ContentType::from_str(content_type) { + Ok(res) => Some(res), + Err(err) => { + error!("parse content-type err {} {:?}", content_type, err); + None + } + } + } else { + None + }; + let last_modified = info.mod_time.map(Timestamp::from); + + if let Some(part_number) = part_number + && rs.is_none() + { + rs = HTTPRangeSpec::from_object_info(&info, part_number); + } + + validate_sse_headers_for_read(&info.user_defined, &req.headers)?; + + let mut content_length = info.get_actual_size().map_err(ApiError::from)?; + let content_range = if let Some(rs) = &rs { + let total_size = content_length; + let (start, length) = rs.get_offset_length(total_size).map_err(ApiError::from)?; + content_length = length; + Some(format!("bytes {}-{}/{}", start, start as i64 + length - 1, total_size)) + } else { + None + }; + + debug!( + "GET object metadata check: parts={}, provided_sse_key={:?}", + info.parts.len(), + req.input.sse_customer_key.is_some() + ); + + let decryption_request = DecryptionRequest { + bucket, + key, + metadata: &info.user_defined, + sse_customer_key: req.input.sse_customer_key.as_ref(), + sse_customer_key_md5: req.input.sse_customer_key_md5.as_ref(), + part_number: None, + parts: &info.parts, + etag: info.etag.as_deref(), + }; + + let mut response_content_length = content_length; + let encrypted_stream = reader.stream; + + let ( + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + encryption_applied, + final_stream, + ) = match sse_decryption(decryption_request).await? { + Some(material) => { + let server_side_encryption = Some(material.server_side_encryption.clone()); + let sse_customer_algorithm = Some(material.algorithm.clone()); + let sse_customer_key_md5 = material.customer_key_md5.clone(); + let ssekms_key_id = material.kms_key_id.clone(); + + let (decrypted_stream, plaintext_size) = material + .wrap_reader(encrypted_stream, content_length) + .await + .map_err(ApiError::from)?; + + response_content_length = plaintext_size; + + ( + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + true, + decrypted_stream, + ) + } + None => (None, None, None, None, false, wrap_reader(encrypted_stream)), + }; + + Ok(GetObjectReadSetup { + info, + event_info, + final_stream, + rs, + content_type, + last_modified, + response_content_length, + content_range, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + encryption_applied, + }) + } + #[allow(clippy::too_many_arguments)] + fn finalize_get_object_strategy( + &self, + manager: &ConcurrencyManager, + bucket: &str, + key: &str, + info: &ObjectInfo, + rs: Option<&HTTPRangeSpec>, + response_content_length: i64, + permit_wait_duration: Duration, + queue_utilization: f64, + queue_status: &concurrency::IoQueueStatus, + concurrent_requests: usize, + ) -> GetObjectStrategyContext { + let base_buffer_size = self.base_buffer_size(); + + let is_sequential_hint = if rs.is_none() { + true + } else if let Some(range_spec) = rs { + range_spec.start == 0 && !range_spec.is_suffix_length + } else { + false + }; + + if let Some(range_spec) = rs + && range_spec.start >= 0 + { + manager.record_access(range_spec.start as u64, response_content_length as u64); + } + + if response_content_length > 0 { + manager.record_transfer(response_content_length as u64, permit_wait_duration); + } + + let io_strategy = + manager.calculate_io_strategy_with_context(info.size, base_buffer_size, permit_wait_duration, is_sequential_hint); + + debug!( + wait_ms = permit_wait_duration.as_millis() as u64, + load_level = ?io_strategy.load_level, + buffer_size = io_strategy.buffer_size, + buffer_multiplier = io_strategy.buffer_multiplier, + readahead = io_strategy.enable_readahead, + storage_media = ?io_strategy.storage_media, + access_pattern = ?io_strategy.access_pattern, + bandwidth_tier = ?io_strategy.bandwidth_tier, + concurrent_requests = io_strategy.concurrent_requests, + file_size = info.size, + is_sequential = is_sequential_hint, + "Enhanced multi-factor I/O strategy calculated" + ); + + let io_priority = manager.get_io_priority(response_content_length); + + if manager.is_priority_scheduling_enabled() { + debug!( + bucket = %bucket, + key = %key, + priority = %io_priority, + request_size = response_content_length, + "I/O priority assigned (based on actual request size)" + ); + + rustfs_io_metrics::record_io_priority_assignment(io_priority.as_str()); + } + + rustfs_io_metrics::record_get_object_io_state( + permit_wait_duration.as_secs_f64(), + queue_utilization, + queue_status.permits_in_use, + queue_status.total_permits.saturating_sub(queue_status.permits_in_use), + io_strategy.load_level.as_str(), + io_strategy.buffer_multiplier, + ); + rustfs_io_metrics::record_io_priority_assignment(io_priority.as_str()); + + debug!( + actual_request_size = response_content_length, + priority = %io_priority.as_str(), + "I/O priority finalized with actual request size" + ); + + let base_buffer_size = get_buffer_size_opt_in(response_content_length); + let optimal_buffer_size = if io_strategy.buffer_size > 0 { + io_strategy.buffer_size.min(base_buffer_size) + } else { + get_concurrency_aware_buffer_size(response_content_length, base_buffer_size) + }; + + debug!( + "GetObject buffer sizing: file_size={}, base={}, optimal={}, concurrent_requests={}, io_strategy={:?}", + response_content_length, base_buffer_size, optimal_buffer_size, concurrent_requests, io_strategy.load_level + ); + + GetObjectStrategyContext { + io_strategy, + optimal_buffer_size, + } + } + + fn build_get_object_checksums( + info: &ObjectInfo, + headers: &HeaderMap, + part_number: Option, + rs: Option<&HTTPRangeSpec>, + ) -> S3Result { + let mut checksums = GetObjectChecksums::default(); + + if let Some(checksum_mode) = headers.get(AMZ_CHECKSUM_MODE) + && checksum_mode.to_str().unwrap_or_default() == "ENABLED" + && rs.is_none() + { + let (decrypted_checksums, _is_multipart) = + info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| { + error!("decrypt_checksums error: {}", e); + ApiError::from(e) + })?; + + for (key, checksum) in decrypted_checksums { + if key == AMZ_CHECKSUM_TYPE { + checksums.checksum_type = Some(ChecksumType::from(checksum)); + continue; + } + + match rustfs_rio::ChecksumType::from_string(key.as_str()) { + rustfs_rio::ChecksumType::CRC32 => checksums.crc32 = Some(checksum), + rustfs_rio::ChecksumType::CRC32C => checksums.crc32c = Some(checksum), + rustfs_rio::ChecksumType::SHA1 => checksums.sha1 = Some(checksum), + rustfs_rio::ChecksumType::SHA256 => checksums.sha256 = Some(checksum), + rustfs_rio::ChecksumType::CRC64_NVME => checksums.crc64nvme = Some(checksum), + _ => (), + } + } + } + + Ok(checksums) + } + #[allow(clippy::too_many_arguments)] + async fn build_get_object_body( + mut final_stream: R, + _info: &ObjectInfo, + response_content_length: i64, + optimal_buffer_size: usize, + part_number: Option, + has_range: bool, + encryption_applied: bool, + ) -> S3Result> + where + R: AsyncRead + Send + Sync + Unpin + 'static, + { + if encryption_applied { + let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD; + let should_buffer_encrypted_object = response_content_length > 0 + && response_content_length <= seekable_object_size_threshold as i64 + && part_number.is_none() + && !has_range; + + if should_buffer_encrypted_object { + let mut buf = Vec::with_capacity(response_content_length as usize); + if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { + error!("Failed to read decrypted object into memory: {}", e); + return Err(ApiError::from(StorageError::other(format!("Failed to read decrypted object: {e}"))).into()); + } + + if buf.len() != response_content_length as usize { + warn!( + "Encrypted object size mismatch during read: expected={} actual={}", + response_content_length, + buf.len() + ); + } + + return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size)); + } + + info!( + "Encrypted object: Using unlimited stream for decryption with buffer size {}", + optimal_buffer_size + ); + return Ok(Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size)); + } + + let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD; + let should_provide_seek_support = response_content_length > 0 + && response_content_length <= seekable_object_size_threshold as i64 + && part_number.is_none() + && !has_range; + + if should_provide_seek_support { + let mut buf = Vec::with_capacity(response_content_length as usize); + match tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { + Ok(_) => { + if buf.len() != response_content_length as usize { + warn!( + "Object size mismatch during seek support read: expected={} actual={}", + response_content_length, + buf.len() + ); + } + + return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size)); + } + Err(e) => { + error!("Failed to read object into memory for seek support: {}", e); + } + } + } + + Ok(Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size)) + } + + fn put_object_execution_context(req: &S3Request) -> (EventName, QuotaOperation, &'static str) { + if req.extensions.get::().is_some() { + (EventName::ObjectCreatedPost, QuotaOperation::PostObject, "POST") + } else { + (EventName::ObjectCreatedPut, QuotaOperation::PutObject, "PUT") + } + } + #[instrument(level = "debug", skip(self, _fs, req))] pub async fn execute_put_object(&self, _fs: &FS, req: S3Request) -> S3Result> { - let request_context = PutObjectRequestContext { - headers: req.headers.clone(), - trailing_headers: req.trailing_headers.clone(), - uri_query: req.uri.query().map(str::to_string), - is_post_object: req.extensions.get::().is_some(), - method: req.method.clone(), - uri: req.uri.clone(), - extensions: req.extensions.clone(), - credentials: req.credentials.clone(), - region: req.region.clone(), - service: req.service.clone(), - }; - let (event_name, quota_operation) = if request_context.is_post_object { - (EventName::ObjectCreatedPost, QuotaOperation::PostObject) - } else { - (EventName::ObjectCreatedPut, QuotaOperation::PutObject) - }; - if request_context.is_post_object && is_post_object_sse_kms_requested(&req.input, &request_context.headers) { + let start_time = std::time::Instant::now(); + let mut req = req; + + if let Some(context) = &self.context { + let _ = context.object_store(); + } + + let (event_name, quota_operation, request_method_name) = Self::put_object_execution_context(&req); + if req.extensions.get::().is_some() && is_post_object_sse_kms_requested(&req.input, &req.headers) + { return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for POST object uploads")); } if let Some(ref storage_class) = req.input.storage_class @@ -632,125 +1416,651 @@ impl DefaultObjectUsecase { { return Err(s3_error!(InvalidStorageClass)); } - if is_put_object_extract_requested(&request_context.headers) { - let helper = OperationHelper::new(&req, event_name, S3Operation::PutObject).suppress_event(); - if is_sse_kms_requested(&req.input, &request_context.headers) { - return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads")); - } - let resolved_size = resolve_put_body_size(req.input.content_length, &request_context.headers)?; - self.check_bucket_quota(&req.input.bucket, quota_operation, resolved_size as u64) - .await?; - let notify = self - .context - .as_ref() - .map(|context| context.notify()) - .unwrap_or_else(default_notify_interface); - let input = req.input; - let output = DefaultObjectUsecase::run_put_object_extract_flow(input, request_context, notify, resolved_size).await?; - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - return result; + if is_put_object_extract_requested(&req.headers) { + return self.execute_put_object_extract(req).await; } - let helper = OperationHelper::new(&req, event_name, S3Operation::PutObject); + let input = std::mem::take(&mut req.input); - let resolved_size = resolve_put_body_size(req.input.content_length, &request_context.headers)?; - self.check_bucket_quota(&req.input.bucket, quota_operation, resolved_size as u64) - .await?; + let PutObjectInput { + body, + bucket, + cache_control, + key, + content_length, + content_disposition, + content_encoding, + content_language, + content_type, + expires, + tagging, + metadata, + version_id, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key, + sse_customer_key_md5, + ssekms_key_id, + content_md5, + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + storage_class, + website_redirect_location, + .. + } = input; - let input = req.input; - let (output, helper_object) = DefaultObjectUsecase::run_put_object_flow(input, request_context, resolved_size).await?; - let helper_version_id = helper_object.version_id.map(|version_id| version_id.to_string()); - let helper = helper.object(helper_object); - let helper = if let Some(version_id) = helper_version_id { - helper.version_id(version_id) - } else { - helper + // Merge SSE-C params from headers (fallback when S3 layer does not populate input) + let (h_algo, h_key, h_md5) = extract_ssec_params_from_headers(&req.headers)?; + let sse_customer_algorithm = sse_customer_algorithm.or(h_algo); + let sse_customer_key = sse_customer_key.or(h_key); + let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5); + + // Merge server_side_encryption from headers (fallback when S3 layer does not populate input) + let server_side_encryption = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); + + // Validate object key + validate_object_key(&key, request_method_name)?; + + if let Some(size) = content_length { + self.check_bucket_quota(&bucket, quota_operation, size as u64).await?; + } + + let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; + + let mut size = match content_length { + Some(c) => c, + None => { + if let Some(val) = req.headers.get(AMZ_DECODED_CONTENT_LENGTH) { + match atoi::atoi::(val.as_bytes()) { + Some(x) => x, + None => return Err(s3_error!(UnexpectedContent)), + } + } else { + return Err(s3_error!(UnexpectedContent)); + } + } }; + + if size == -1 { + return Err(s3_error!(UnexpectedContent)); + } + + // Apply adaptive buffer sizing based on file size for optimal streaming performance. + // Uses workload profile configuration (enabled by default) to select appropriate buffer size. + // Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile. + let buffer_size = get_buffer_size_opt_in(size); + + // Detect zero-copy opportunity before encryption/compression decisions + // Zero-copy is beneficial for large unencrypted, uncompressed objects + let enable_zero_copy = should_use_zero_copy(size, &req.headers); + + if enable_zero_copy { + // Record zero-copy write attempt + counter!("rustfs.zero_copy.write.attempts.total").increment(1); + histogram!("rustfs.zero_copy.write.size.bytes").record(size as f64); + debug!("Zero-copy write enabled for {} byte object (bucket={}, key={})", size, bucket, key); + } + + let body = tokio::io::BufReader::with_capacity( + buffer_size, + StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), + ); + + let store = get_validated_store(&bucket).await?; + + // TDD: Get bucket default encryption configuration + let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + debug!("TDD: bucket_sse_config={:?}", bucket_sse_config); + + // TDD: Determine effective encryption configuration (request overrides bucket default) + let original_sse = server_side_encryption.clone(); + let mut effective_sse = server_side_encryption.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + debug!("TDD: Processing bucket SSE config: {:?}", config); + config.rules.first().and_then(|rule| { + debug!("TDD: Processing SSE rule: {:?}", rule); + rule.apply_server_side_encryption_by_default.as_ref().map(|sse| { + debug!("TDD: Found SSE default: {:?}", sse); + match sse.sse_algorithm.as_str() { + "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), + _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), // fallback to AES256 + } + }) + }) + }) + }); + debug!("TDD: effective_sse={:?} (original={:?})", effective_sse, original_sse); + + let mut effective_kms_key_id = ssekms_key_id.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + config.rules.first().and_then(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .and_then(|sse| sse.kms_master_key_id.clone()) + }) + }) + }); + + // Validate SSE-C headers early: reject partial/invalid combinations per S3 spec + validate_sse_headers_for_write( + effective_sse.as_ref(), + effective_kms_key_id.as_ref(), + sse_customer_algorithm.as_ref(), + sse_customer_key.as_ref(), + sse_customer_key_md5.as_ref(), + true, // PutObject requires all three: algorithm, key, key_md5 + )?; + + let mut metadata = metadata.unwrap_or_default(); + apply_put_request_metadata( + &mut metadata, + &req.headers, + &key, + cache_control, + content_disposition, + content_encoding, + content_language, + content_type, + expires, + website_redirect_location, + tagging, + storage_class.clone(), + )?; + + let mut opts: ObjectOptions = put_opts(&bucket, &key, version_id.clone(), &req.headers, metadata.clone()) + .await + .map_err(ApiError::from)?; + apply_put_request_object_lock_opts( + &bucket, + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + &mut opts, + ) + .await?; + + let current_opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers) + .await + .map_err(ApiError::from)?; + match store.get_object_info(&bucket, &key, ¤t_opts).await { + Ok(existing_obj_info) => validate_existing_object_lock_for_write(&existing_obj_info)?, + Err(err) => { + if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { + return Err(ApiError::from(err).into()); + } + } + } + + let actual_size = size; + + let mut md5hex = if let Some(base64_md5) = content_md5 { + let md5 = base64_simd::STANDARD + .decode_to_vec(base64_md5.as_bytes()) + .map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?; + Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower)) + } else { + None + }; + + let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); + + let mut reader = if is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64 { + let algorithm = CompressionAlgorithm::default(); + insert_str(&mut metadata, SUFFIX_COMPRESSION, algorithm.to_string()); + insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); + + let mut hrd = + HashReader::from_stream(body, size, size, md5hex.take(), sha256hex.take(), false).map_err(ApiError::from)?; + + if let Err(err) = hrd.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) { + return Err(ApiError::from(err).into()); + } + + opts.want_checksum = hrd.checksum(); + insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, algorithm.to_string()); + insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string()); + + size = HashReader::SIZE_PRESERVE_LAYER; + HashReader::from_reader(CompressReader::new(hrd, algorithm), size, actual_size, None, None, false) + .map_err(ApiError::from)? + } else { + HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? + }; + + if size >= 0 { + if let Err(err) = reader.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) { + return Err(ApiError::from(err).into()); + } + + opts.want_checksum = reader.checksum(); + } + + let mut helper = OperationHelper::new(&req, event_name, S3Operation::PutObject); + + // Apply encryption using unified SSE API. + let encryption_request = EncryptionRequest { + bucket: &bucket, + key: &key, + server_side_encryption: effective_sse.clone(), + ssekms_key_id: effective_kms_key_id.clone(), + sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key, + sse_customer_key_md5: sse_customer_key_md5.clone(), + content_size: actual_size, + part_number: None, + part_key: None, + part_nonce: None, + }; + + let encryption_material = match sse_encryption(encryption_request).await { + Ok(material) => material, + Err(err) => { + let result = Err(err.into()); + let _ = helper.complete(&result); + return result; + } + }; + + if let Some(material) = encryption_material { + effective_sse = Some(material.server_side_encryption.clone()); + effective_kms_key_id = material.kms_key_id.clone(); + + let encrypted_reader = material.wrap_reader(reader); + reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + .map_err(ApiError::from)?; + + let encryption_metadata = material.metadata; + metadata.extend(encryption_metadata.clone()); + opts.user_defined.extend(encryption_metadata); + } + + let mut reader = PutObjReader::new(reader); + + let mt2 = metadata.clone(); + opts.user_defined.extend(metadata); + + let repoptions = + get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts.clone()); + + let dsc = must_replicate(&bucket, &key, repoptions).await; + + if dsc.replicate_any() { + insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); + insert_str( + &mut opts.user_defined, + SUFFIX_REPLICATION_STATUS, + dsc.pending_status().unwrap_or_default(), + ); + } + + let obj_info = match store + .put_object(&bucket, &key, &mut reader, &opts) + .await + .map_err(ApiError::from) + { + Ok(obj_info) => obj_info, + Err(err) => { + let result: S3Result> = Err(err.into()); + let _ = helper.complete(&result); + return result; + } + }; + + maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; + + // Fast in-memory update for immediate quota consistency + rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, obj_info.size as u64).await; + + let raw_version = obj_info.version_id.map(|v| v.to_string()); + + helper = helper.object(obj_info.clone()); + if let Some(version_id) = &raw_version { + helper = helper.version_id(version_id.clone()); + } + + let put_version = if BucketVersioningSys::prefix_enabled(&bucket, &key).await { + raw_version + } else { + None + }; + + let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); + + let repoptions = + get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts); + + let dsc = must_replicate(&bucket, &key, repoptions).await; + let expiration = resolve_put_object_expiration(&bucket, &obj_info).await; + + if dsc.replicate_any() { + schedule_replication(obj_info.clone(), store, dsc, ReplicationType::Object).await; + } + + let mut checksums = PutObjectChecksums { + crc32: input.checksum_crc32, + crc32c: input.checksum_crc32c, + sha1: input.checksum_sha1, + sha256: input.checksum_sha256, + crc64nvme: input.checksum_crc64nvme, + }; + apply_trailing_checksums( + input.checksum_algorithm.as_ref().map(|a| a.as_str()), + &req.trailing_headers, + &mut checksums, + ); + + let output = PutObjectOutput { + e_tag, + server_side_encryption: effective_sse, + sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key_md5: sse_customer_key_md5.clone(), + ssekms_key_id: effective_kms_key_id, + expiration, + checksum_crc32: checksums.crc32, + checksum_crc32c: checksums.crc32c, + checksum_sha1: checksums.sha1, + checksum_sha256: checksums.sha256, + checksum_crc64nvme: checksums.crc64nvme, + version_id: put_version, + ..Default::default() + }; + + // For browser-based POST uploads (multipart/form-data), response status/body handling + // is decided by s3s PostObject serializer (success_action_status / redirect semantics). + let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); + + // Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead) + let manager = get_capacity_manager(); + manager.record_write_operation().await; + + // Record PutObject metrics via zero-copy-metrics + { + let duration_ms = start_time.elapsed().as_millis() as f64; + rustfs_io_metrics::record_put_object( + duration_ms, + size, + enable_zero_copy, // Track if zero-copy was enabled + ); + } + result } + fn finalize_get_object_completion( + wrapper: &RequestTimeoutWrapper, + timeout_config: &TimeoutConfig, + total_duration: Duration, + response_content_length: i64, + optimal_buffer_size: usize, + ) { + rustfs_io_metrics::record_get_object_completion( + total_duration.as_secs_f64(), + response_content_length, + optimal_buffer_size, + ); + + rustfs_io_metrics::record_get_object(total_duration.as_millis() as f64, response_content_length); + + if wrapper.is_timeout() { + warn!( + "GetObject request exceeded timeout: duration={:?} timeout={:?}", + wrapper.elapsed(), + timeout_config.get_object_timeout + ); + rustfs_io_metrics::record_get_object_timeout(None, Some(wrapper.elapsed().as_secs_f64())); + } + + debug!( + "GetObject completed: size={} duration={:?} buffer={}", + response_content_length, total_duration, optimal_buffer_size + ); + } + + async fn finalize_get_object_response( + helper: OperationHelper, + bucket: &str, + method: &hyper::Method, + headers: &HeaderMap, + event_info: ObjectInfo, + version_id_for_event: String, + output: GetObjectOutput, + ) -> S3Result> { + let helper = helper.object(event_info).version_id(version_id_for_event); + let response = wrap_response_with_cors(bucket, method, headers, output).await; + let result = Ok(response); + let _ = helper.complete(&result); + result + } + #[allow(clippy::too_many_arguments)] + async fn build_get_object_output_context( + &self, + req: &S3Request, + manager: &ConcurrencyManager, + bucket: &str, + key: &str, + info: ObjectInfo, + event_info: ObjectInfo, + final_stream: DynReader, + rs: Option, + content_type: Option, + last_modified: Option, + response_content_length: i64, + content_range: Option, + server_side_encryption: Option, + sse_customer_algorithm: Option, + sse_customer_key_md5: Option, + ssekms_key_id: Option, + encryption_applied: bool, + permit_wait_duration: Duration, + queue_utilization: f64, + queue_status: &concurrency::IoQueueStatus, + concurrent_requests: usize, + part_number: Option, + versioned: bool, + ) -> S3Result { + let strategy = self.finalize_get_object_strategy( + manager, + bucket, + key, + &info, + rs.as_ref(), + response_content_length, + permit_wait_duration, + queue_utilization, + queue_status, + concurrent_requests, + ); + let GetObjectStrategyContext { + io_strategy: _, + optimal_buffer_size, + } = strategy; + + let body = Self::build_get_object_body( + final_stream, + &info, + response_content_length, + optimal_buffer_size, + part_number, + rs.is_some(), + encryption_applied, + ) + .await?; + + let checksums = Self::build_get_object_checksums(&info, &req.headers, part_number, rs.as_ref())?; + + let output_version_id = if versioned { + info.version_id.map(|vid| { + if vid == Uuid::nil() { + "null".to_string() + } else { + vid.to_string() + } + }) + } else { + None + }; + + let output = GetObjectOutput { + body, + content_length: Some(response_content_length), + last_modified, + content_type, + content_encoding: info.content_encoding.clone(), + accept_ranges: Some("bytes".to_string()), + content_range, + e_tag: info.etag.map(|etag| to_s3s_etag(&etag)), + metadata: filter_object_metadata(&info.user_defined), + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + checksum_crc32: checksums.crc32, + checksum_crc32c: checksums.crc32c, + checksum_sha1: checksums.sha1, + checksum_sha256: checksums.sha256, + checksum_crc64nvme: checksums.crc64nvme, + checksum_type: checksums.checksum_type, + version_id: output_version_id, + ..Default::default() + }; + + Ok(GetObjectOutputContext { + output, + event_info, + response_content_length, + optimal_buffer_size, + }) + } + #[instrument( level = "debug", skip(self, req), fields(start_time=?time::OffsetDateTime::now_utc()) )] pub async fn execute_get_object(&self, req: S3Request) -> S3Result> { + if let Some(context) = &self.context { + let _ = context.object_store(); + } + let request_id = req .extensions - .get::() + .get::() .map(|ctx| ctx.request_id.clone()) - .unwrap_or_else(|| request_context::RequestContext::fallback().request_id); - let bootstrap = { - let timeout_config = TimeoutConfig::from_env(); - let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.clone()); - let request_start = std::time::Instant::now(); - let request_guard = crate::storage::concurrency::ConcurrencyManager::track_request(); - let concurrent_requests = GetObjectGuard::concurrent_requests(); + .unwrap_or_else(|| crate::storage::request_context::RequestContext::fallback().request_id); + let bootstrap = Self::init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?; + let timeout_config = bootstrap.timeout_config; + let wrapper = bootstrap.wrapper; + let request_start = bootstrap.request_start; + let concurrent_requests = bootstrap.concurrent_requests; + let mut request_guard = bootstrap.request_guard; - let deadlock_detector = deadlock_detector::get_deadlock_detector(); - deadlock_detector.register_request(&request_id, format!("GetObject {}/{}", req.input.bucket, req.input.key)); - let deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector, request_id); - - if wrapper.is_timeout() { - warn!( - bucket = %req.input.bucket, - key = %req.input.key, - timeout_secs = timeout_config.get_object_timeout.as_secs(), - elapsed_ms = wrapper.elapsed().as_millis(), - "GetObject request timed out before processing" - ); - return Err(s3_error!(InternalError, "Request timeout before processing")); - } - - rustfs_io_metrics::record_get_object_request_start(concurrent_requests); - - debug!( - "GetObject request started with {} concurrent requests, timeout={:?}", - concurrent_requests, timeout_config.get_object_timeout - ); - - GetObjectBootstrap { - timeout_config, - wrapper, - request_start, - request_guard, - _deadlock_request_guard: deadlock_request_guard, - } - }; - let version_id_for_event = req.input.version_id.clone().unwrap_or_default(); - let request_context = prepare_get_object_request_context(&req).await?; - let base_buffer_size = self.base_buffer_size(); - let manager = get_concurrency_manager(); - let cors_bucket = request_context.bucket.clone(); - let cors_method = req.method.clone(); - let cors_headers = request_context.headers.clone(); let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).suppress_event(); - let flow_result = - get_object_flow::run_get_object_flow(request_context, version_id_for_event, manager, &bootstrap, base_buffer_size) - .await; + // mc get 3 - let GetObjectBootstrap { - mut request_guard, - _deadlock_request_guard, - .. - } = bootstrap; + let request_context = Self::prepare_get_object_request_context(&req).await?; + let GetObjectRequestContext { + bucket, + key, + version_id_for_event, + part_number, + rs, + opts, + } = request_context; - let result = match flow_result { - Ok(flow_result) => { - let helper = helper - .object(flow_result.event_info) - .version_id(flow_result.version_id_for_event); - let response = wrap_response_with_cors(&cors_bucket, &cors_method, &cors_headers, flow_result.output).await; - let result = Ok(response); - let _ = helper.complete(&result); - result - } - Err(err) => Err(err), - }; + // Try to get from cache for small, frequently accessed objects + let manager = get_concurrency_manager(); + let prepared_read = Self::prepare_get_object_read_execution( + &req, + manager, + &wrapper, + &timeout_config, + &bucket, + &key, + rs, + &opts, + part_number, + ) + .await?; + let GetObjectPreparedRead { io_planning, read_setup } = prepared_read; + let permit_wait_duration = io_planning.permit_wait_duration; + let queue_status = io_planning.queue_status; + let queue_utilization = io_planning.queue_utilization; + + let GetObjectReadSetup { + info, + event_info, + final_stream, + rs, + content_type, + last_modified, + response_content_length, + content_range, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + encryption_applied, + } = read_setup; + + let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; + let output_context = self + .build_get_object_output_context( + &req, + manager, + &bucket, + &key, + info, + event_info, + final_stream, + rs, + content_type, + last_modified, + response_content_length, + content_range, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + encryption_applied, + permit_wait_duration, + queue_utilization, + &queue_status, + concurrent_requests, + part_number, + versioned, + ) + .await?; + let GetObjectOutputContext { + output, + event_info, + response_content_length, + optimal_buffer_size, + } = output_context; + + let total_duration = request_start.elapsed(); + Self::finalize_get_object_completion( + &wrapper, + &timeout_config, + total_duration, + response_content_length, + optimal_buffer_size, + ); + + let result = Self::finalize_get_object_response( + helper, + &bucket, + &req.method, + &req.headers, + event_info, + version_id_for_event, + output, + ) + .await; if result.is_ok() { request_guard.finish_ok(); } else { @@ -763,6 +2073,10 @@ impl DefaultObjectUsecase { &self, req: S3Request, ) -> S3Result> { + if let Some(context) = &self.context { + let _ = context.object_store(); + } + let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedAttributes, S3Operation::GetObjectAttributes).suppress_event(); let GetObjectAttributesInput { @@ -990,6 +2304,10 @@ impl DefaultObjectUsecase { #[instrument(level = "debug", skip(self, req))] pub async fn execute_copy_object(&self, req: S3Request) -> S3Result> { + if let Some(context) = &self.context { + let _ = context.object_store(); + } + let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedCopy, S3Operation::CopyObject); let CopyObjectInput { copy_source, @@ -1140,8 +2458,6 @@ impl DefaultObjectUsecase { src_info.metadata_only = true; } - let mut reader: Box = Box::new(WarpReader::new(gr.stream)); - let decryption_request = DecryptionRequest { bucket: &src_bucket, key: &src_key, @@ -1153,11 +2469,12 @@ impl DefaultObjectUsecase { etag: src_info.etag.as_deref(), }; - if let Some(material) = sse_decryption(decryption_request).await? { - reader = material.wrap_single_reader(reader); - if let Some(original) = material.original_size { - src_info.actual_size = original; - } + let decryption_material = sse_decryption(decryption_request).await?; + + if let Some(material) = decryption_material.as_ref() + && let Some(original) = material.original_size + { + src_info.actual_size = original; } strip_managed_encryption_metadata(&mut src_info.user_defined); @@ -1168,16 +2485,11 @@ impl DefaultObjectUsecase { let mut compress_metadata = HashMap::new(); - if is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64 { + let should_compress = is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64; + + if should_compress { insert_str(&mut compress_metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); - - let hrd = EtagReader::new(reader, None); - - // let hrd = HashReader::new(reader, length, actual_size, None, false).map_err(ApiError::from)?; - - reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default())); - length = HashReader::SIZE_PRESERVE_LAYER; } else { remove_str(&mut src_info.user_defined, SUFFIX_COMPRESSION); remove_str(&mut src_info.user_defined, SUFFIX_ACTUAL_SIZE); @@ -1194,7 +2506,7 @@ impl DefaultObjectUsecase { } if let Some(ct) = content_type { src_info.content_type = Some(ct.clone()); - src_info.user_defined.insert(CONTENT_TYPE.to_string(), ct); + src_info.user_defined.insert("content-type".to_string(), ct); } } @@ -1209,7 +2521,68 @@ impl DefaultObjectUsecase { src_info.user_defined.extend(object_lock_metadata); } - let mut reader = HashReader::new(reader, length, actual_size, None, None, false).map_err(ApiError::from)?; + let mut reader = match decryption_material { + Some(material) => { + if material.is_multipart { + let (decrypted_stream, plaintext_size) = + material.wrap_reader(gr.stream, length).await.map_err(ApiError::from)?; + length = plaintext_size; + + if should_compress { + let hrd = HashReader::from_reader(decrypted_stream, length, actual_size, None, None, false) + .map_err(ApiError::from)?; + length = HashReader::SIZE_PRESERVE_LAYER; + HashReader::from_reader( + CompressReader::new(hrd, CompressionAlgorithm::default()), + length, + actual_size, + None, + None, + false, + ) + .map_err(ApiError::from)? + } else { + HashReader::from_reader(decrypted_stream, length, actual_size, None, None, false) + .map_err(ApiError::from)? + } + } else if should_compress { + let hrd = + HashReader::from_stream(material.wrap_single_reader(gr.stream), length, actual_size, None, None, false) + .map_err(ApiError::from)?; + length = HashReader::SIZE_PRESERVE_LAYER; + HashReader::from_reader( + CompressReader::new(hrd, CompressionAlgorithm::default()), + length, + actual_size, + None, + None, + false, + ) + .map_err(ApiError::from)? + } else { + HashReader::from_stream(material.wrap_single_reader(gr.stream), length, actual_size, None, None, false) + .map_err(ApiError::from)? + } + } + None => { + if should_compress { + let hrd = + HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)?; + length = HashReader::SIZE_PRESERVE_LAYER; + HashReader::from_reader( + CompressReader::new(hrd, CompressionAlgorithm::default()), + length, + actual_size, + None, + None, + false, + ) + .map_err(ApiError::from)? + } else { + HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)? + } + } + }; let encryption_request = EncryptionRequest { bucket: &bucket, @@ -1230,7 +2603,7 @@ impl DefaultObjectUsecase { effective_kms_key_id = material.kms_key_id.clone(); let encrypted_reader = material.wrap_reader(reader); - reader = HashReader::new(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) .map_err(ApiError::from)?; src_info.user_defined.extend(material.metadata); @@ -1253,7 +2626,7 @@ impl DefaultObjectUsecase { .await .map_err(ApiError::from)?; - enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await; + maybe_enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await; // Update quota tracking after successful copy if has_bucket_metadata { @@ -1298,6 +2671,10 @@ impl DefaultObjectUsecase { &self, mut req: S3Request, ) -> S3Result> { + if let Some(context) = &self.context { + let _ = context.object_store(); + } + let helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObjects).suppress_event(); let (bucket, delete) = { let bucket = req.input.bucket.clone(); @@ -1331,7 +2708,6 @@ impl DefaultObjectUsecase { let version_cfg = BucketVersioningSys::get(&bucket).await.unwrap_or_default(); let bypass_governance = has_bypass_governance_header(&req.headers); - let capacity_scope_token = Uuid::new_v4(); #[derive(Default, Clone)] struct DeleteResult { @@ -1454,12 +2830,14 @@ impl DefaultObjectUsecase { object_to_delete.clone(), ObjectOptions { version_suspended: version_cfg.suspended(), - capacity_scope_token: Some(capacity_scope_token), ..Default::default() }, ) .await; + let _manager = get_concurrency_manager(); + let _bucket_clone = bucket.clone(); + let _deleted_objects = dobjs.clone(); if is_all_buckets_not_found( &errs .iter() @@ -1593,12 +2971,17 @@ impl DefaultObjectUsecase { let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); // Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead) - record_capacity_write(Some(capacity_scope_token)).await; + let manager = get_capacity_manager(); + manager.record_write_operation().await; result } #[instrument(level = "debug", skip(self, req))] pub async fn execute_delete_object(&self, mut req: S3Request) -> S3Result> { + if let Some(context) = &self.context { + let _ = context.object_store(); + } + let mut helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObject); let DeleteObjectInput { bucket, key, version_id, .. @@ -1624,8 +3007,6 @@ impl DefaultObjectUsecase { let mut opts: ObjectOptions = del_opts(&bucket, &key, version_id, &req.headers, metadata) .await .map_err(ApiError::from)?; - let capacity_scope_token = Uuid::new_v4(); - opts.capacity_scope_token = Some(capacity_scope_token); let force_delete = opts.delete_prefix; let lock_cfg = BucketObjectLockSys::get(&bucket).await; @@ -1728,6 +3109,7 @@ impl DefaultObjectUsecase { }) .await; } + // Prefix/force-delete returns empty ObjectInfo; still emit bucket notification so webhooks match S3 DELETE. helper = helper .event_name(EventName::ObjectRemovedDelete) .object(ObjectInfo { @@ -1737,7 +3119,9 @@ impl DefaultObjectUsecase { }) .version_id(String::new()); let result = Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT)); - record_capacity_write(Some(capacity_scope_token)).await; + // Match non-empty delete path: capacity manager write-op telemetry. + let manager = get_capacity_manager(); + manager.record_write_operation().await; let _ = helper.complete(&result); return result; } @@ -1785,13 +3169,18 @@ impl DefaultObjectUsecase { let result = Ok(S3Response::new(output)); // Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead) - record_capacity_write(Some(capacity_scope_token)).await; + let manager = get_capacity_manager(); + manager.record_write_operation().await; let _ = helper.complete(&result); result } #[instrument(level = "debug", skip(self, req))] pub async fn execute_head_object(&self, req: S3Request) -> S3Result> { + if let Some(context) = &self.context { + let _ = context.object_store(); + } + let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedHead, S3Operation::HeadObject).suppress_event(); // mc get 2 let HeadObjectInput { @@ -2114,6 +3503,10 @@ impl DefaultObjectUsecase { #[instrument(level = "debug", skip(self, req))] pub async fn execute_restore_object(&self, req: S3Request) -> S3Result> { + if let Some(context) = &self.context { + let _ = context.object_store(); + } + let mut helper = OperationHelper::new(&req, EventName::ObjectRestorePost, S3Operation::RestoreObject); let RestoreObjectInput { bucket, @@ -2262,7 +3655,7 @@ impl DefaultObjectUsecase { let rreq_clone = rreq.clone(); let version_id_clone = version_id.clone(); - request_context::spawn_traced(async move { + spawn_traced(async move { let opts = ObjectOptions { transition: TransitionOptions { restore_request: rreq_clone, @@ -2303,6 +3696,10 @@ impl DefaultObjectUsecase { &self, req: S3Request, ) -> S3Result> { + if let Some(context) = &self.context { + let _ = context.object_store(); + } + info!("handle select_object_content"); let input = Arc::new(req.input); @@ -2354,7 +3751,7 @@ impl DefaultObjectUsecase { let (tx, rx) = mpsc::channel::>(2); let stream = ReceiverStream::new(rx); - request_context::spawn_traced(async move { + spawn_traced(async move { let _ = tx .send(Ok(SelectObjectContentEvent::Cont(ContinuationEvent::default()))) .await; @@ -2372,6 +3769,419 @@ impl DefaultObjectUsecase { payload: Some(SelectObjectContentEventStream::new(stream)), })) } + + #[instrument(level = "debug", skip(self, req))] + pub async fn execute_put_object_extract(&self, req: S3Request) -> S3Result> { + let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, S3Operation::PutObject).suppress_event(); + let auth_method = req.method.clone(); + let auth_uri = req.uri.clone(); + let auth_headers = req.headers.clone(); + let auth_extensions = req.extensions.clone(); + let auth_credentials = req.credentials.clone(); + let auth_region = req.region.clone(); + let auth_service = req.service.clone(); + let auth_trailing_headers = req.trailing_headers.clone(); + if is_sse_kms_requested(&req.input, &req.headers) { + return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads")); + } + let input = req.input; + + let PutObjectInput { + body, + bucket, + key, + version_id, + cache_control, + content_disposition, + content_encoding, + content_length, + content_language, + content_type, + content_md5, + expires, + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key, + sse_customer_key_md5, + ssekms_key_id, + storage_class, + tagging, + website_redirect_location, + .. + } = input; + + let event_version_id = version_id; + let (h_algo, h_key, h_md5) = extract_ssec_params_from_headers(&req.headers)?; + let sse_customer_algorithm = sse_customer_algorithm.or(h_algo); + let sse_customer_key = sse_customer_key.or(h_key); + let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5); + + let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); + let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + let mut effective_sse = original_sse.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + config.rules.first().and_then(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .map(|sse| match sse.sse_algorithm.as_str() { + "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), + _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + }) + }) + }) + }); + let mut effective_kms_key_id = ssekms_key_id.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + config.rules.first().and_then(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .and_then(|sse| sse.kms_master_key_id.clone()) + }) + }) + }); + if effective_sse + .as_ref() + .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) + { + return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads")); + } + validate_sse_headers_for_write( + effective_sse.as_ref(), + effective_kms_key_id.as_ref(), + sse_customer_algorithm.as_ref(), + sse_customer_key.as_ref(), + sse_customer_key_md5.as_ref(), + true, + )?; + let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; + + let size = match content_length { + Some(c) => c, + None => { + if let Some(val) = req.headers.get(AMZ_DECODED_CONTENT_LENGTH) { + match atoi::atoi::(val.as_bytes()) { + Some(x) => x, + None => return Err(s3_error!(UnexpectedContent)), + } + } else { + return Err(s3_error!(UnexpectedContent)); + } + } + }; + if size == -1 { + return Err(s3_error!(UnexpectedContent)); + } + validate_object_key(&key, "PUT")?; + self.check_bucket_quota(&bucket, QuotaOperation::PutObject, size as u64) + .await?; + + // Apply adaptive buffer sizing based on file size for optimal streaming performance. + // Uses workload profile configuration (enabled by default) to select appropriate buffer size. + // Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile. + let buffer_size = get_buffer_size_opt_in(size); + let body = tokio::io::BufReader::with_capacity( + buffer_size, + StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), + ); + + let Some(ext) = Path::new(&key).extension().and_then(|s| s.to_str()) else { + return Err(s3_error!(InvalidArgument, "key extension not found")); + }; + + let ext = ext.to_owned(); + + let md5hex = if let Some(base64_md5) = content_md5 { + let md5 = base64_simd::STANDARD + .decode_to_vec(base64_md5.as_bytes()) + .map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?; + Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower)) + } else { + None + }; + + let sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); + let actual_size = size; + + let mut archive_reader = + HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; + + if let Err(err) = archive_reader.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) { + return Err(ApiError::from(err).into()); + } + + let archive_etag = Arc::new(Mutex::new(None)); + let decoder = CompressionFormat::from_extension(&ext) + .get_decoder(ExtractArchiveEtagReader::new(archive_reader, archive_etag.clone())) + .map_err(|e| { + error!("get_decoder err {:?}", e); + s3_error!(InvalidArgument, "get_decoder err") + })?; + + let mut ar = Archive::new(decoder); + let mut entries = ar.entries().map_err(|e| { + error!("get entries err {:?}", e); + s3_error!(InvalidArgument, "get entries err") + })?; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let extract_options = resolve_put_object_extract_options(&req.headers); + let version_id = match event_version_id { + Some(v) => v.to_string(), + None => String::new(), + }; + + let notify = self + .context + .as_ref() + .map(|context| context.notify()) + .unwrap_or_else(default_notify_interface); + let req_params = extract_params_header(&req.headers); + let host = get_request_host(&req.headers); + let port = get_request_port(&req.headers); + let user_agent = get_request_user_agent(&req.headers); + + while let Some(entry) = entries.next().await { + let mut f = match entry { + Ok(f) => f, + Err(e) => { + if extract_options.ignore_errors { + warn!("Skipping archive entry because read failed and ignore-errors is enabled: {e}"); + continue; + } + error!("Failed to read archive entry: {}", e); + return Err(s3_error!(InvalidArgument, "Failed to read archive entry: {:?}", e)); + } + }; + + let fpath = match f.path() { + Ok(path) => path, + Err(e) => { + if extract_options.ignore_errors { + warn!("Skipping archive entry because path decode failed and ignore-errors is enabled: {e}"); + continue; + } + return Err(s3_error!(InvalidArgument, "Failed to decode archive entry path")); + } + }; + + let is_dir = f.header().entry_type().is_dir(); + let fpath = normalize_extract_entry_key(&fpath.to_string_lossy(), extract_options.prefix.as_deref(), is_dir); + + let mut auth_req = S3Request { + input: PutObjectInput::default(), + method: auth_method.clone(), + uri: auth_uri.clone(), + headers: auth_headers.clone(), + extensions: auth_extensions.clone(), + credentials: auth_credentials.clone(), + region: auth_region.clone(), + service: auth_service.clone(), + trailing_headers: auth_trailing_headers.clone(), + }; + { + let req_info = req_info_mut(&mut auth_req)?; + req_info.bucket = Some(bucket.clone()); + req_info.object = Some(fpath.clone()); + req_info.version_id = None; + } + authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectAction)).await?; + + let mut size = f.header().size().unwrap_or_default() as i64; + let archive_entry_mod_time = f + .header() + .mtime() + .ok() + .and_then(|modified_at_secs| OffsetDateTime::from_unix_timestamp(modified_at_secs as i64).ok()); + let mut metadata = HashMap::new(); + apply_put_request_metadata( + &mut metadata, + &req.headers, + &fpath, + cache_control.clone(), + content_disposition.clone(), + content_encoding.clone(), + content_language.clone(), + content_type.clone(), + expires.clone(), + website_redirect_location.clone(), + tagging.clone(), + storage_class.clone(), + )?; + let mut opts = put_opts(&bucket, &fpath, None, &req.headers, metadata.clone()) + .await + .map_err(ApiError::from)?; + apply_extract_entry_pax_extensions(&mut f, &mut metadata, &mut opts).await?; + if archive_entry_mod_time.is_some() { + opts.mod_time = archive_entry_mod_time; + } + + debug!("Extracting file: {}, size: {} bytes", fpath, size); + + if is_dir { + if extract_options.ignore_dirs { + debug!("Skipping directory entry during archive extract: {}", fpath); + continue; + } + size = 0; + } + + let actual_size = size; + + let should_compress = !is_dir && is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64; + + let mut hrd = if is_dir { + HashReader::from_stream(std::io::Cursor::new(Vec::new()), size, actual_size, None, None, false) + .map_err(ApiError::from)? + } else if should_compress { + insert_str(&mut metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); + insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); + + let hrd = HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)?; + size = HashReader::SIZE_PRESERVE_LAYER; + HashReader::from_reader( + CompressReader::new(hrd, CompressionAlgorithm::default()), + size, + actual_size, + None, + None, + false, + ) + .map_err(ApiError::from)? + } else { + HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)? + }; + apply_put_request_object_lock_opts( + &bucket, + object_lock_legal_hold_status.clone(), + object_lock_mode.clone(), + object_lock_retain_until_date.clone(), + &mut opts, + ) + .await?; + if let Some(material) = sse_encryption(EncryptionRequest { + bucket: &bucket, + key: &fpath, + server_side_encryption: effective_sse.clone(), + ssekms_key_id: effective_kms_key_id.clone(), + sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key: sse_customer_key.clone(), + sse_customer_key_md5: sse_customer_key_md5.clone(), + content_size: actual_size, + part_number: None, + part_key: None, + part_nonce: None, + }) + .await? + { + effective_sse = Some(material.server_side_encryption.clone()); + effective_kms_key_id = material.kms_key_id.clone(); + + let encrypted_reader = material.wrap_reader(hrd); + hrd = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + .map_err(ApiError::from)?; + + let encryption_metadata = material.metadata; + metadata.extend(encryption_metadata.clone()); + opts.user_defined.extend(encryption_metadata); + } + opts.user_defined.extend(metadata); + let mut reader = PutObjReader::new(hrd); + + let obj_info = match store.put_object(&bucket, &fpath, &mut reader, &opts).await { + Ok(info) => info, + Err(e) => { + if extract_options.ignore_errors { + warn!("Skipping archive entry because object write failed and ignore-errors is enabled: {e}"); + continue; + } + return Err(ApiError::from(e).into()); + } + }; + + let _manager = get_concurrency_manager(); + let _fpath_clone = fpath.clone(); + let _bucket_clone = bucket.clone(); + let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); + + let output = PutObjectOutput { + e_tag, + ..Default::default() + }; + + let event_args = rustfs_notify::EventArgs { + event_name: EventName::ObjectCreatedPut, + bucket_name: bucket.clone(), + object: obj_info.clone(), + req_params: req_params.clone(), + resp_elements: extract_resp_elements(&S3Response::new(output.clone())), + version_id: version_id.clone(), + host: host.clone(), + port, + user_agent: user_agent.clone(), + }; + + let notify = notify.clone(); + let request_context = req + .extensions + .get::() + .cloned(); + spawn_background_with_context(request_context, async move { + notify.notify(event_args).await; + }); + } + + let mut checksums = PutObjectChecksums { + crc32: input.checksum_crc32, + crc32c: input.checksum_crc32c, + sha1: input.checksum_sha1, + sha256: input.checksum_sha256, + crc64nvme: input.checksum_crc64nvme, + }; + apply_trailing_checksums( + input.checksum_algorithm.as_ref().map(|a| a.as_str()), + &req.trailing_headers, + &mut checksums, + ); + + warn!( + "put object extract checksum_crc32={:?}, checksum_crc32c={:?}, checksum_sha1={:?}, checksum_sha256={:?}, checksum_crc64nvme={:?}", + checksums.crc32, checksums.crc32c, checksums.sha1, checksums.sha256, checksums.crc64nvme, + ); + + drop(entries); + let mut decoder = match ar.into_inner() { + Ok(decoder) => decoder, + Err(_) => return Err(s3_error!(InvalidArgument, "Failed to finalize archive reader")), + }; + tokio::io::copy(&mut decoder, &mut tokio::io::sink()) + .await + .map_err(map_extract_archive_error)?; + let archive_etag = archive_etag + .lock() + .ok() + .and_then(|etag| etag.clone()) + .map(|etag| to_s3s_etag(&etag)); + + let output = PutObjectOutput { + e_tag: archive_etag, + checksum_crc32: checksums.crc32, + checksum_crc32c: checksums.crc32c, + checksum_sha1: checksums.sha1, + checksum_sha256: checksums.sha256, + checksum_crc64nvme: checksums.crc64nvme, + ..Default::default() + }; + let result = Ok(S3Response::new(output)); + let _ = helper.complete(&result); + result + } } fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'static str) -> bool { @@ -2386,7 +4196,7 @@ fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'s #[cfg(test)] mod tests { use super::*; - use http::{Extensions, HeaderMap, Method, Uri}; + use http::{Extensions, HeaderMap, HeaderName, HeaderValue, Method, Uri}; fn build_request(input: T, method: Method) -> S3Request { S3Request { @@ -2402,6 +4212,292 @@ mod tests { } } + #[test] + fn is_put_object_extract_requested_accepts_meta_header() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + + assert!(is_put_object_extract_requested(&headers)); + } + + #[test] + fn is_put_object_extract_requested_accepts_compat_header_case_insensitive() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_EXTRACT_COMPAT, HeaderValue::from_static(" TRUE ")); + + assert!(is_put_object_extract_requested(&headers)); + } + + #[test] + fn is_put_object_extract_requested_rejects_missing_or_false_value() { + let mut headers = HeaderMap::new(); + assert!(!is_put_object_extract_requested(&headers)); + + headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("false")); + assert!(!is_put_object_extract_requested(&headers)); + } + + #[test] + fn normalize_snowball_prefix_trims_slashes_and_whitespace() { + assert_eq!(normalize_snowball_prefix(" /batch/incoming/ "), Some("batch/incoming".to_string())); + assert_eq!(normalize_snowball_prefix("///"), None); + } + + #[test] + fn normalize_extract_entry_key_applies_prefix_and_directory_suffix() { + assert_eq!( + normalize_extract_entry_key("nested/path.txt", Some("imports"), false), + "imports/nested/path.txt" + ); + assert_eq!(normalize_extract_entry_key("nested/dir/", Some("imports"), true), "imports/nested/dir/"); + assert_eq!(normalize_extract_entry_key("top-level", None, false), "top-level"); + } + + #[test] + fn should_use_zero_copy_rejects_boundary_at_1mb() { + let headers = HeaderMap::new(); + + assert!(!should_use_zero_copy(1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_small_objects() { + let headers = HeaderMap::new(); + + assert!(!should_use_zero_copy(1024 * 1024 - 1, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_one_megabyte() { + let headers = HeaderMap::new(); + + assert!(!should_use_zero_copy(1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_encrypted_requests() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("AES256")); + + assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_encrypted_requests_with_sse_customer_algorithm() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, HeaderValue::from_static("AES256")); + + assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_encrypted_requests_with_kms_key_id() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, HeaderValue::from_static("test-kms-key-id")); + + assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_compressible_content_types() { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json; charset=utf-8")); + + assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_allows_large_unencrypted_binary_objects() { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); + + assert!(should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn resolve_put_object_extract_options_defaults_when_headers_missing() { + let headers = HeaderMap::new(); + let options = resolve_put_object_extract_options(&headers); + assert_eq!( + options, + PutObjectExtractOptions { + prefix: None, + ignore_dirs: false, + ignore_errors: false + } + ); + } + + #[test] + fn resolve_put_object_extract_options_accepts_internal_headers() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_PREFIX_INTERNAL, HeaderValue::from_static("/internal/prefix/")); + headers.insert(AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, HeaderValue::from_static("true")); + headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, HeaderValue::from_static("TRUE")); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("internal/prefix")); + assert!(options.ignore_dirs); + assert!(options.ignore_errors); + } + + #[test] + fn resolve_put_object_extract_options_accepts_standard_headers() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static(" /standard/prefix/ ")); + headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static(" true ")); + headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("TRUE")); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("standard/prefix")); + assert!(options.ignore_dirs); + assert!(options.ignore_errors); + } + + #[test] + fn resolve_put_object_extract_options_accepts_suffix_compatible_headers() { + let mut headers = HeaderMap::new(); + headers.insert( + HeaderName::from_static("x-amz-meta-acme-snowball-prefix"), + HeaderValue::from_static(" /partner/import "), + ); + headers.insert( + HeaderName::from_static("x-amz-meta-acme-snowball-ignore-dirs"), + HeaderValue::from_static(" true "), + ); + headers.insert( + HeaderName::from_static("x-amz-meta-acme-snowball-ignore-errors"), + HeaderValue::from_static("TRUE"), + ); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("partner/import")); + assert!(options.ignore_dirs); + assert!(options.ignore_errors); + } + + #[test] + fn resolve_put_object_extract_options_prefers_exact_headers_over_suffix_fallback() { + let mut headers = HeaderMap::new(); + headers.insert("x-amz-meta-acme-snowball-prefix", HeaderValue::from_static("/fallback/prefix/")); + headers.insert(AMZ_RUSTFS_SNOWBALL_PREFIX, HeaderValue::from_static("/internal/prefix/")); + headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static("/standard/prefix/")); + headers.insert(AMZ_MINIO_SNOWBALL_PREFIX, HeaderValue::from_static("/minio/prefix/")); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("minio/prefix")); + } + + #[test] + fn resolve_put_object_extract_options_exact_flags_override_suffix_fallback() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static("false")); + headers.insert("x-amz-meta-acme-snowball-ignore-dirs", HeaderValue::from_static("true")); + headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("false")); + headers.insert("x-amz-meta-acme-snowball-ignore-errors", HeaderValue::from_static("true")); + + let options = resolve_put_object_extract_options(&headers); + assert!(!options.ignore_dirs); + assert!(!options.ignore_errors); + } + + #[tokio::test] + async fn execute_put_object_rejects_post_object_sse_kms_from_input() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .server_side_encryption(Some(ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS))) + .build() + .unwrap(); + + let mut req = build_request(input, Method::POST); + req.extensions.insert(PostObjectRequestMarker); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + #[tokio::test] + async fn execute_put_object_rejects_extract_sse_kms() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("archive.tar".to_string()) + .server_side_encryption(Some(ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS))) + .build() + .unwrap(); + + let mut req = build_request(input, Method::PUT); + req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + #[tokio::test] + async fn execute_put_object_extract_rejects_invalid_storage_class() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("archive.tar".to_string()) + .storage_class(Some(StorageClass::from_static("INVALID"))) + .build() + .unwrap(); + + let mut req = build_request(input, Method::PUT); + req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass); + } + + #[tokio::test] + async fn execute_put_object_rejects_post_object_sse_kms_from_headers() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .build() + .unwrap(); + + let mut req = build_request(input, Method::POST); + req.extensions.insert(PostObjectRequestMarker); + req.headers + .insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("aws:kms")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + #[tokio::test] + async fn execute_put_object_rejects_post_object_sse_kms_key_id_header() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .build() + .unwrap(); + + let mut req = build_request(input, Method::POST); + req.extensions.insert(PostObjectRequestMarker); + req.headers + .insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, HeaderValue::from_static("test-kms-key-id")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + #[tokio::test] async fn execute_put_object_rejects_invalid_storage_class() { let input = PutObjectInput::builder() @@ -2419,31 +4515,20 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass); } - #[test] - fn normalize_delete_objects_version_id_handles_null_uuid_and_empty_values() { - let (raw, uuid) = normalize_delete_objects_version_id(Some("null".to_string())).unwrap(); - assert_eq!(raw.as_deref(), Some("null")); - assert_eq!(uuid, Some(Uuid::nil())); + #[tokio::test] + async fn execute_get_object_rejects_zero_part_number() { + let input = GetObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .part_number(Some(0)) + .build() + .unwrap(); - let (raw, uuid) = normalize_delete_objects_version_id(Some(String::new())).unwrap(); - assert!(raw.is_none()); - assert!(uuid.is_none()); + let req = build_request(input, Method::GET); + let usecase = DefaultObjectUsecase::without_context(); - let (raw, uuid) = normalize_delete_objects_version_id(Some(" ".to_string())).unwrap(); - assert!(raw.is_none()); - assert!(uuid.is_none()); - - let valid = "550e8400-e29b-41d4-a716-446655440000".to_string(); - let (raw, uuid) = normalize_delete_objects_version_id(Some(valid.clone())).unwrap(); - assert_eq!(raw.as_deref(), Some(valid.as_str())); - assert_eq!(uuid, Some(Uuid::parse_str(&valid).unwrap())); - - let err = normalize_delete_objects_version_id(Some("not-a-uuid".to_string())).unwrap_err(); - assert!(!err.is_empty()); - - let (raw, uuid) = normalize_delete_objects_version_id(None).unwrap(); - assert!(raw.is_none()); - assert!(uuid.is_none()); + let err = usecase.execute_get_object(req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); } #[tokio::test] diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index d2e164748..14a2d8352 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -257,24 +257,8 @@ impl From for ApiError { impl From for ApiError { fn from(err: std::io::Error) -> Self { - // Check if the inner error is a StorageError (e.g. InvalidRangeSpec wrapped by object-io) + // Check if the error is a ChecksumMismatch (BadDigest) if let Some(inner) = err.get_ref() { - if let Some(storage_error) = inner.downcast_ref::() { - let code = match storage_error { - StorageError::InvalidRangeSpec(_) => S3ErrorCode::InvalidRange, - _ => S3ErrorCode::InternalError, - }; - let message = if code == S3ErrorCode::InternalError { - storage_error.to_string() - } else { - ApiError::error_code_to_message(&code) - }; - return ApiError { - code, - message, - source: Some(Box::new(err)), - }; - } if inner.downcast_ref::().is_some() { return ApiError { code: S3ErrorCode::BadDigest,