From 5ce884f605030815a55b1518777927867f57c887 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 25 Aug 2026 13:01:53 +0800 Subject: [PATCH] chore(deps): update s3s revision (#6545) * chore(deps): update s3s revision Pin the workspace s3s dependency to rustfs/s3s commit 39080d610e0560c55f068f6dd76b976e267b2f67 and refresh compatible dependencies with cargo update and cargo upgrade. Co-authored-by: heihutu * fix(s3): preserve SigV4 body validation errors Map s3s upload stream body validation failures into existing RustFS client-error types before the PUT body readers consume them. This keeps tampered single-chunk payload hashes from surfacing as InternalError after the s3s revision update. Co-Authored-By: heihutu * chore(deps): use s3s 0.15.0 release Switch the workspace dependency from the temporary s3s git revision to the published 0.15.0 crate and refresh the lockfile updates that come with the release. Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- Cargo.lock | 146 ++++++++++++++++++++----------- Cargo.toml | 6 +- rustfs/src/app/object_usecase.rs | 45 +++++++--- 3 files changed, 130 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88a510f37..f299083b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -794,7 +794,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -1685,27 +1685,25 @@ dependencies = [ [[package]] name = "bon" -version = "3.9.3" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" dependencies = [ "bon-macros", - "rustversion", ] [[package]] name = "bon-macros" -version = "3.9.3" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "ident_case", - "prettyplease", + "prettyplease 0.3.0", "proc-macro2", "quote", - "rustversion", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] @@ -2052,7 +2050,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -2633,6 +2631,16 @@ dependencies = [ "darling_macro 0.23.0", ] +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + [[package]] name = "darling_core" version = "0.20.11" @@ -2673,6 +2681,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.4", +] + [[package]] name = "darling_macro" version = "0.20.11" @@ -2706,6 +2727,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.4", +] + [[package]] name = "dary_heap" version = "0.3.9" @@ -3204,7 +3236,7 @@ checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" dependencies = [ "datafusion-doc", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -3803,7 +3835,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -4385,7 +4417,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -4788,9 +4820,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -6316,9 +6348,9 @@ dependencies = [ [[package]] name = "metrique" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dedbf06ffeef4c37990c73636fbd993aa34fb1948afd736e6114f239220993db" +checksum = "2b55bfa39e6f5e44a37a59a794915ce36d04371471cf62ae365cd9703f58a5e0" dependencies = [ "itoa", "jiff", @@ -6346,9 +6378,9 @@ dependencies = [ [[package]] name = "metrique-macro" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fb1f30185f53f7f6e4c9e46745c1a1350af8e77fda5a88aded44b0637a82e0" +checksum = "7605c49951713cce25af6bee77ed4432dde0839c5d9935bcb6f9643e6e82c7c3" dependencies = [ "Inflector", "darling 0.23.0", @@ -6375,9 +6407,9 @@ checksum = "2faca4e4480069ff02b1763b3b79f5cec7e8628e24d9dc5b6073f53d2577a4d9" [[package]] name = "metrique-writer" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20bd17c1a3ca2719e31f19ce77a853948dc2102f35976b92276c42a64fdc5f3f" +checksum = "bb97854dd4e2a92fe54b30e6e19976dcc0b2b7d39da3e84e86e2ce1e2ed8ce0d" dependencies = [ "ahash", "crossbeam-queue", @@ -6396,9 +6428,9 @@ dependencies = [ [[package]] name = "metrique-writer-core" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a55b6aae1d85c557c729564c4e2b32a26dc65ba2d90d9647ca01f2bd4854c4" +checksum = "71c9fc241cff4eedc2b6c3e2edece7f3edab2920e7531f62aac864ce933cdbb6" dependencies = [ "derive-where", "itertools 0.14.0", @@ -6931,7 +6963,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.22.1", + "base64 0.21.7", "chrono", "getrandom 0.2.17", "http 1.5.0", @@ -7948,6 +7980,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.4", +] + [[package]] name = "prettytable-rs" version = "0.10.0" @@ -8098,13 +8140,13 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", + "heck 0.4.1", + "itertools 0.10.5", "log", "multimap", "once_cell", "petgraph 0.7.1", - "prettyplease", + "prettyplease 0.2.37", "prost 0.13.5", "prost-types 0.13.5", "regex", @@ -8118,12 +8160,12 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", + "heck 0.4.1", + "itertools 0.10.5", "log", "multimap", "petgraph 0.8.3", - "prettyplease", + "prettyplease 0.2.37", "prost 0.14.4", "prost-types 0.14.4", "pulldown-cmark", @@ -8140,7 +8182,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.119", @@ -8153,7 +8195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.119", @@ -8219,12 +8261,13 @@ dependencies = [ [[package]] name = "pulsar" -version = "6.8.0" +version = "6.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2784d3ed4cc69abf1257842f40b5ca89cfa2e6047912a9691193532c211ab13" +checksum = "5a56d908fe26714fbd8dffbe6a69b39ea09fb28af14fdd8f971548d2906346b3" dependencies = [ "async-channel", "async-trait", + "base64 0.22.1", "bytes", "chrono", "crc", @@ -8705,7 +8748,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -8983,9 +9026,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.63.0" +version = "0.63.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00cf00190c315093734a8d405225bd8773a219bc86538a9b73bfc51145b33995" +checksum = "35bab1b87d915817d5d9cc352637cd40d5f0b298a48c6309af9156a4addc3031" dependencies = [ "aes 0.9.2", "aws-lc-rs", @@ -10758,8 +10801,9 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" -version = "0.15.0-alpha.1" -source = "git+https://github.com/rustfs/s3s.git?rev=e080e38c56a3b43acbacce55710d765a5ce9003d#e080e38c56a3b43acbacce55710d765a5ce9003d" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "652451a66fefda01a13dc7df24aa2af9e70c7058f98bc08fe2f7c552eaa9f1e3" dependencies = [ "arc-swap", "arrayvec", @@ -11025,7 +11069,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -11089,7 +11133,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -11169,7 +11213,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -11766,9 +11810,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" dependencies = [ "proc-macro2", "quote", @@ -11891,7 +11935,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -11987,7 +12031,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -12133,7 +12177,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -12315,7 +12359,7 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" dependencies = [ - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "syn 2.0.119", @@ -12338,7 +12382,7 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "prost-build 0.14.4", "prost-types 0.14.4", @@ -13459,7 +13503,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 026cebb43..4df0e590c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -147,7 +147,7 @@ futures-core = "0.3.34" futures-lite = "2.6.1" futures-util = "0.3.34" pollster = "1.0.1" -pulsar = { default-features = false, version = "6.8.0" } +pulsar = { default-features = false, version = "6.9.0" } lapin = { default-features = false, version = "4.10.0" } hyper = { version = "1.11.0" } hyper-rustls = { default-features = false, version = "0.27.9" } @@ -293,7 +293,7 @@ rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.4" } rust-embed = { version = "8.12.0" } rustc-hash = { version = "2.1.3" } -s3s = { git = "https://github.com/rustfs/s3s.git", rev = "e080e38c56a3b43acbacce55710d765a5ce9003d" } +s3s = { version = "0.15.0", features = ["minio"] } serial_test = "4.0.1" shadow-rs = { default-features = false, version = "2.0.0" } siphasher = "1.0.3" @@ -345,7 +345,7 @@ libunftp = { version = "0.23.0" } unftp-core = "0.1.0" suppaftp = { version = "10.0.2" } rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] } -russh = { version = "0.63.0" } +russh = { version = "0.63.1" } russh-sftp = "2.4.0" # WebDAV diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 380b86ef7..ababda1cd 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -190,6 +190,7 @@ const PUT_EAGER_STATUS_ZERO_COPY_INELIGIBLE: &str = "zero_copy_ineligible"; const PUT_EAGER_STATUS_AWS_CHUNKED_MISSING_DECODED_LENGTH: &str = "aws_chunked_missing_decoded_length"; static CACHED_ZERO_COPY_EAGER_PUT_MAX_SIZE_BYTES: std::sync::OnceLock = std::sync::OnceLock::new(); use std::collections::HashMap; +use std::io; use std::ops::Add; use std::path::Path; use std::pin::Pin; @@ -233,6 +234,26 @@ use crate::app::object_traffic_health::ObjectTrafficHealth; type S3StdError = Box; +fn s3s_body_error_to_io(err: StdError) -> io::Error { + match err.to_string().as_str() { + "UploadStreamError: Sha256Mismatch" => { + return io::Error::new( + io::ErrorKind::InvalidData, + rustfs_rio::ChecksumMismatch { + want: AMZ_CONTENT_SHA256.to_string(), + got: "payload sha256".to_string(), + }, + ); + } + "UploadStreamError: Incomplete" | "UploadStreamError: LengthMismatch" => { + return io::Error::new(io::ErrorKind::UnexpectedEof, rustfs_rio::IncompleteBody { remaining: 0 }); + } + _ => {} + } + + io::Error::other(err) +} + struct ColdFillDiskPermitMetric { owner: ColdFillDiskPermitOwner, metric_recorded: bool, @@ -2821,7 +2842,7 @@ where let mut remaining = (&mut *buf).limit(size - filled); let read = tokio::io::AsyncReadExt::read_buf(&mut *body, &mut remaining) .await - .map_err(|err| ApiError::from(StorageError::other(err.to_string())))?; + .map_err(ApiError::from)?; if read == 0 { return Err(s3_error!(IncompleteBody)); } @@ -2831,7 +2852,7 @@ where let mut extra = [0u8; 1]; let extra_read = tokio::io::AsyncReadExt::read(&mut *body, &mut extra) .await - .map_err(|err| ApiError::from(StorageError::other(err.to_string())))?; + .map_err(ApiError::from)?; if extra_read != 0 { return Err(s3_error!(UnexpectedContent)); } @@ -2863,7 +2884,7 @@ where async fn read_zero_copy_put_body_exact(mut body: S, size: usize) -> S3Result where S: futures::Stream> + Unpin, - E: std::fmt::Display, + E: Into, { let mut chunks = Vec::new(); let mut filled = 0usize; @@ -2872,7 +2893,7 @@ where let Some(chunk) = body.next().await else { return Err(s3_error!(IncompleteBody)); }; - let chunk = chunk.map_err(|err| ApiError::from(StorageError::other(err.to_string())))?; + let chunk = chunk.map_err(|err| ApiError::from(s3s_body_error_to_io(err.into())))?; if chunk.is_empty() { continue; } @@ -2886,7 +2907,7 @@ where } while let Some(chunk) = body.next().await { - let chunk = chunk.map_err(|err| ApiError::from(StorageError::other(err.to_string())))?; + let chunk = chunk.map_err(|err| ApiError::from(s3s_body_error_to_io(err.into())))?; if !chunk.is_empty() { return Err(s3_error!(UnexpectedContent)); } @@ -6172,7 +6193,7 @@ impl DefaultObjectUsecase { let mut reader = if should_compress { let body = tokio::io::BufReader::with_capacity( buffer_size, - StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), + StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), ); let algorithm = CompressionAlgorithm::default(); insert_str(&mut metadata, SUFFIX_COMPRESSION, compression_metadata_value(algorithm)); @@ -6205,7 +6226,7 @@ impl DefaultObjectUsecase { // Mutex contention under high concurrency. Direct allocation // for ≤4KiB is negligible cost. let eager_body = read_small_put_body_exact_direct( - StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), + StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), actual_size as usize, ) .await?; @@ -6213,7 +6234,7 @@ impl DefaultObjectUsecase { } else { let pool = get_concurrency_manager().bytes_pool(); let eager_body = read_small_put_body_exact_pooled( - StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), + StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), actual_size as usize, pool.as_ref(), ) @@ -6224,7 +6245,7 @@ impl DefaultObjectUsecase { } else { let body = tokio::io::BufReader::with_capacity( buffer_size, - StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), + StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), ); HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? } @@ -9784,10 +9805,8 @@ impl DefaultObjectUsecase { // Uses workload profile configuration (enabled by default) to select appropriate buffer size. // Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile. let buffer_size = get_buffer_size_opt_in(size); - let body = tokio::io::BufReader::with_capacity( - buffer_size, - StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), - ); + let body = + tokio::io::BufReader::with_capacity(buffer_size, StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io)))); let Some(ext) = Path::new(&key).extension().and_then(|s| s.to_str()) else { return Err(s3_error!(InvalidArgument, "key extension not found"));