From 8e4a1ef91759e0e8143ac34ca5a774ff7217c793 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 8 Mar 2026 15:18:15 +0800 Subject: [PATCH] refactor(protocols): replace tar with astral-tokio-tar for async processing (#2099) Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> --- Cargo.lock | 115 +++++++----------- Cargo.toml | 14 +-- crates/protocols/Cargo.toml | 16 +-- crates/protocols/src/swift/bulk.rs | 143 ++++++++++++++++++++--- crates/protocols/src/swift/encryption.rs | 4 +- crates/protocols/src/swift/expiration.rs | 10 +- crates/protocols/src/swift/formpost.rs | 6 +- crates/protocols/src/swift/handler.rs | 39 +++---- crates/protocols/src/swift/symlink.rs | 6 +- crates/protocols/src/swift/tempurl.rs | 14 +++ crates/protocols/src/swift/versioning.rs | 14 +++ 11 files changed, 235 insertions(+), 146 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 916574ee7..f06ac34e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -613,9 +613,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.14" +version = "1.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2" +checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -633,7 +633,7 @@ dependencies = [ "fastrand", "hex", "http 1.4.0", - "ring", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -643,9 +643,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.13" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d203b0bf2626dcba8665f5cd0871d7c2c0930223d6b6be9097592fea21242d0" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -678,9 +678,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede2ddc593e6c8acc6ce3358c28d6677a6dc49b65ba4b37a2befe14a11297e75" +checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -706,9 +706,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.124.0" +version = "1.125.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744c09d75dfec039a05cf8e117c995ded3b0baffa6eb83f3ed7075a01d8d8947" +checksum = "223f5c95650d9557925a91f4c2db3def189e8f659452134a29e5cd2d37d708ed" dependencies = [ "aws-credential-types", "aws-runtime", @@ -741,9 +741,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.95.0" +version = "1.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00c5ff27c6ba2cbd95e6e26e2e736676fdf6bcf96495b187733f521cfe4ce448" +checksum = "f64a6eded248c6b453966e915d32aeddb48ea63ad17932682774eb026fbef5b1" dependencies = [ "aws-credential-types", "aws-runtime", @@ -765,9 +765,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.97.0" +version = "1.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d186f1e5a3694a188e5a0640b3115ccc6e084d104e16fd6ba968dca072ffef8" +checksum = "db96d720d3c622fcbe08bae1c4b04a72ce6257d8b0584cb5418da00ae20a344f" dependencies = [ "aws-credential-types", "aws-runtime", @@ -789,9 +789,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.99.0" +version = "1.100.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9acba7c62f3d4e2408fa998a3a8caacd8b9a5b5549cf36e2372fbdae329d5449" +checksum = "fafbdda43b93f57f699c5dfe8328db590b967b8a820a13ccdd6687355dfcc7ca" dependencies = [ "aws-credential-types", "aws-runtime", @@ -814,9 +814,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37411f8e0f4bea0c3ca0958ce7f18f6439db24d555dbd809787262cd00926aa9" +checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -1036,9 +1036,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.13" +version = "1.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0470cc047657c6e286346bdf10a8719d26efd6a91626992e0e64481e44323e96" +checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1311,16 +1311,6 @@ dependencies = [ "bytes", ] -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - [[package]] name = "bzip2" version = "0.6.1" @@ -1330,16 +1320,6 @@ dependencies = [ "libbz2-rs-sys", ] -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "camino" version = "1.2.2" @@ -1490,7 +1470,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.6", + "crypto-common 0.1.7", "inout 0.1.4", ] @@ -1604,7 +1584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" dependencies = [ "brotli", - "bzip2 0.6.1", + "bzip2", "compression-core", "flate2", "liblzma", @@ -1961,9 +1941,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", @@ -2232,7 +2212,7 @@ dependencies = [ "arrow-schema", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -2370,7 +2350,7 @@ dependencies = [ "async-compression", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -3045,7 +3025,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.6", + "crypto-common 0.1.7", "subtle", ] @@ -3626,9 +3606,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -6225,9 +6205,9 @@ checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24" [[package]] name = "pprof-pyroscope-fork" -version = "0.1500.2" +version = "0.1500.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa86bc766c4e3e07526f21028ad9cdd112903c2cc646b36724c54978fb42f81" +checksum = "79f090659c6f3f8c12785dabca0cb7f40154a40dc254c06848598ff31c3be1d8" dependencies = [ "aligned-vec", "backtrace", @@ -7748,12 +7728,12 @@ dependencies = [ name = "rustfs-protocols" version = "0.0.5" dependencies = [ + "astral-tokio-tar", + "async-compression", "async-trait", "axum", "base64 0.22.1", "bytes", - "bzip2 0.4.4", - "flate2", "futures", "futures-util", "hex", @@ -7761,13 +7741,10 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper", "libunftp", "md5", "percent-encoding", - "pin-project-lite", "quick-xml 0.39.2", - "rand 0.10.0", "regex", "rustfs-credentials", "rustfs-ecstore", @@ -7782,7 +7759,6 @@ dependencies = [ "serde_json", "sha1 0.11.0-rc.5", "sha2 0.11.0-rc.5", - "tar", "thiserror 2.0.18", "time", "tokio", @@ -8213,7 +8189,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.14.0-dev" -source = "git+https://github.com/s3s-project/s3s?rev=2fa67f48764a0d80f304304d34c3f978e0fdacf5#2fa67f48764a0d80f304304d34c3f978e0fdacf5" +source = "git+https://github.com/s3s-project/s3s?rev=4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990#4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990" dependencies = [ "arc-swap", "arrayvec", @@ -8831,12 +8807,12 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9142,17 +9118,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" -[[package]] -name = "tar" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "temp-env" version = "0.3.6" @@ -9852,13 +9817,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.21.0" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" dependencies = [ "getrandom 0.4.2", "js-sys", - "rand 0.9.2", + "rand 0.10.0", "serde_core", "wasm-bindgen", ] @@ -10777,7 +10742,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b680f2a0cd479b4cff6e1233c483fdead418106eae419dc60200ae9850f6d004" dependencies = [ "aes 0.8.4", - "bzip2 0.6.1", + "bzip2", "constant_time_eq", "crc32fast", "deflate64", diff --git a/Cargo.toml b/Cargo.toml index e1459fccc..88df5d250 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,7 +122,7 @@ http = "1.4.0" http-body = "1.0.1" http-body-util = "0.1.3" reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] } -socket2 = { version = "0.6.2", features = ["all"] } +socket2 = { version = "0.6.3", features = ["all"] } tokio = { version = "1.50.0", features = ["fs", "rt-multi-thread"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] } tokio-stream = { version = "0.1.18" } @@ -180,9 +180,9 @@ arc-swap = "1.8.2" astral-tokio-tar = "0.5.6" atoi = "2.0.0" atomic_enum = "0.3.0" -aws-config = { version = "1.8.14" } -aws-credential-types = { version = "1.2.13" } -aws-sdk-s3 = { version = "1.124.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } +aws-config = { version = "1.8.15" } +aws-credential-types = { version = "1.2.14" } +aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] } aws-smithy-types = { version = "1.4.6" } backtrace = "0.3.76" @@ -237,7 +237,7 @@ rumqttc = { version = "0.25.1" } rustix = { version = "1.1.4", features = ["fs"] } rust-embed = { version = "8.11.0" } rustc-hash = { version = "2.1.1" } -s3s = { git = "https://github.com/s3s-project/s3s", rev = "2fa67f48764a0d80f304304d34c3f978e0fdacf5", features = ["minio"] } +s3s = { git = "https://github.com/s3s-project/s3s", rev = "4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990", features = ["minio"] } serial_test = "3.4.0" shadow-rs = { version = "1.7.0", default-features = false } siphasher = "1.0.2" @@ -260,7 +260,7 @@ tracing-subscriber = { version = "0.3.22", features = ["env-filter", "time"] } transform-stream = "0.3.1" url = "2.5.8" urlencoding = "2.1.3" -uuid = { version = "1.21.0", features = ["v4", "fast-rng", "macro-diagnostics"] } +uuid = { version = "1.22.0", features = ["v4", "fast-rng", "macro-diagnostics"] } vaultrs = { version = "0.7.4" } walkdir = "2.5.0" wildmatch = { version = "2.6.1", features = ["serde"] } @@ -296,7 +296,7 @@ jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] } # Used to generate CPU performance analysis data and flame diagrams # pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] } # Pyroscope uses a patched pprof, until they merge back upstream, replace all references. Otherwise, two pprof libs with symbol collision. -pprof = { package = "pprof-pyroscope-fork", version = "0.1500.2", features = ["flamegraph", "protobuf-codec"] } +pprof = { package = "pprof-pyroscope-fork", version = "0.1500.3", features = ["flamegraph", "protobuf-codec"] } [workspace.metadata.cargo-shear] ignored = ["rustfs", "rustfs-mcp"] diff --git a/crates/protocols/Cargo.toml b/crates/protocols/Cargo.toml index 9788065ec..3b13f16c3 100644 --- a/crates/protocols/Cargo.toml +++ b/crates/protocols/Cargo.toml @@ -34,13 +34,11 @@ swift = [ "dep:rustfs-rio", "dep:axum", "dep:http", - "dep:hyper", "dep:tower", "dep:regex", "dep:percent-encoding", "dep:sha2", "dep:uuid", - "dep:pin-project-lite", "dep:futures", "dep:http-body", "dep:http-body-util", @@ -52,11 +50,9 @@ swift = [ "dep:hmac", "dep:sha1", "dep:hex", - "dep:tar", - "dep:flate2", - "dep:bzip2", + "dep:astral-tokio-tar", "dep:base64", - "dep:rand", + "dep:async-compression", ] [dependencies] @@ -96,13 +92,11 @@ rustfs-ecstore = { workspace = true, optional = true } rustfs-rio = { workspace = true, optional = true } axum = { workspace = true, optional = true } http = { workspace = true, optional = true } -hyper = { workspace = true, optional = true } tower = { workspace = true, optional = true } regex = { workspace = true, optional = true } percent-encoding = { workspace = true, optional = true } sha2 = { workspace = true, optional = true } uuid = { workspace = true, optional = true } -pin-project-lite = { workspace = true, optional = true } futures = { workspace = true, optional = true } http-body = { workspace = true, optional = true } http-body-util = { workspace = true, optional = true } @@ -114,11 +108,9 @@ quick-xml = { workspace = true, optional = true, features = ["serialize"] } hmac = { workspace = true, optional = true } sha1 = { workspace = true, optional = true } hex = { workspace = true, optional = true } -tar = { version = "0.4", optional = true } -flate2 = { workspace = true, optional = true } -bzip2 = { version = "0.4", optional = true } +astral-tokio-tar = { workspace = true, optional = true } base64 = { workspace = true, optional = true } -rand = { workspace = true, optional = true } +async-compression = { workspace = true, optional = true, features = ["tokio", "gzip", "bzip2"] } [package.metadata.docs.rs] all-features = true diff --git a/crates/protocols/src/swift/bulk.rs b/crates/protocols/src/swift/bulk.rs index d31fb5ac3..2d93a4d8f 100644 --- a/crates/protocols/src/swift/bulk.rs +++ b/crates/protocols/src/swift/bulk.rs @@ -76,10 +76,10 @@ use super::{SwiftError, SwiftResult, container, object}; use axum::http::{Response, StatusCode}; +use futures::StreamExt; use rustfs_credentials::Credentials; use s3s::Body; use serde::{Deserialize, Serialize}; -use std::io::Read; use tracing::{debug, error}; /// Result of a single delete operation @@ -340,8 +340,8 @@ pub async fn handle_bulk_extract( return Err(SwiftError::NotFound(format!("Container not found: {}", container))); } - // Parse archive and collect entries (without holding the archive) - let entries = extract_tar_entries(format, body)?; + // Parse archive and collect all entries into memory (entire archive and file contents are buffered) + let entries = extract_tar_entries(format, body).await?; // Now upload each entry (async operations) for (path_str, contents) in entries { @@ -352,7 +352,7 @@ pub async fn handle_bulk_extract( &path_str, credentials, std::io::Cursor::new(contents), - &axum::http::HeaderMap::new(), + &http::HeaderMap::new(), ) .await { @@ -395,30 +395,30 @@ pub async fn handle_bulk_extract( .map_err(|e| SwiftError::InternalServerError(format!("Failed to build response: {}", e))) } -/// Extract tar entries synchronously to avoid Send issues -fn extract_tar_entries(format: ArchiveFormat, body: Vec) -> SwiftResult)>> { +/// Extract tar entries using async I/O and return them as in-memory buffers +async fn extract_tar_entries(format: ArchiveFormat, body: Vec) -> SwiftResult)>> { // Create appropriate reader based on format - let reader: Box = match format { + let reader: Box = match format { ArchiveFormat::Tar => Box::new(std::io::Cursor::new(body)), ArchiveFormat::TarGz => { let cursor = std::io::Cursor::new(body); - Box::new(flate2::read::GzDecoder::new(cursor)) + Box::new(async_compression::tokio::bufread::GzipDecoder::new(tokio::io::BufReader::new(cursor))) } ArchiveFormat::TarBz2 => { let cursor = std::io::Cursor::new(body); - Box::new(bzip2::read::BzDecoder::new(cursor)) + Box::new(async_compression::tokio::bufread::BzDecoder::new(tokio::io::BufReader::new(cursor))) } }; // Parse tar archive - let mut archive = tar::Archive::new(reader); + let mut archive = tokio_tar::Archive::new(reader); let mut entries = Vec::new(); + let mut entries_iter = archive + .entries() + .map_err(|e| SwiftError::BadRequest(format!("Failed to read tar archive: {}", e)))?; // Extract each entry - for entry in archive - .entries() - .map_err(|e| SwiftError::BadRequest(format!("Failed to read tar archive: {}", e)))? - { + while let Some(entry) = entries_iter.next().await { let mut entry = entry.map_err(|e| SwiftError::BadRequest(format!("Failed to read tar entry: {}", e)))?; // Get entry path @@ -436,7 +436,7 @@ fn extract_tar_entries(format: ArchiveFormat, body: Vec) -> SwiftResult Vec { + let buf = std::io::Cursor::new(Vec::new()); + let mut builder = tokio_tar::Builder::new(buf); + + for &dir in dirs { + let mut header = tokio_tar::Header::new_gnu(); + header.set_entry_type(tokio_tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o755); + header.set_cksum(); + builder + .append_data(&mut header, dir, std::io::Cursor::new(&[] as &[u8])) + .await + .unwrap(); + } + + for &(name, data) in files { + let mut header = tokio_tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, name, std::io::Cursor::new(data)) + .await + .unwrap(); + } + + builder.into_inner().await.unwrap().into_inner() + } + + #[tokio::test] + async fn test_extract_plain_tar_paths_and_contents() { + let tar_bytes = make_tar(&[("file1.txt", b"hello"), ("dir/file2.txt", b"world")], &[]).await; + let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].0, "file1.txt"); + assert_eq!(entries[0].1, b"hello"); + assert_eq!(entries[1].0, "dir/file2.txt"); + assert_eq!(entries[1].1, b"world"); + } + + #[tokio::test] + async fn test_extract_tar_skips_directories() { + let tar_bytes = make_tar(&[("file.txt", b"content")], &["subdir/", "another/"]).await; + let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap(); + // Directory entries must be filtered out; only the regular file is returned. + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].0, "file.txt"); + } + + #[tokio::test] + async fn test_extract_tar_gz() { + let tar_bytes = make_tar(&[("file.txt", b"compressed content")], &[]).await; + + // Compress with gzip. + let cursor = std::io::Cursor::new(Vec::new()); + let mut encoder = async_compression::tokio::write::GzipEncoder::new(cursor); + encoder.write_all(&tar_bytes).await.unwrap(); + encoder.shutdown().await.unwrap(); + let gz_bytes = encoder.into_inner().into_inner(); + + let entries = extract_tar_entries(ArchiveFormat::TarGz, gz_bytes).await.unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].0, "file.txt"); + assert_eq!(entries[0].1, b"compressed content"); + } + + #[tokio::test] + async fn test_extract_tar_bz2() { + let tar_bytes = make_tar(&[("file.txt", b"bzip2 content")], &[]).await; + + // Compress with bzip2. + let cursor = std::io::Cursor::new(Vec::new()); + let mut encoder = async_compression::tokio::write::BzEncoder::new(cursor); + encoder.write_all(&tar_bytes).await.unwrap(); + encoder.shutdown().await.unwrap(); + let bz2_bytes = encoder.into_inner().into_inner(); + + let entries = extract_tar_entries(ArchiveFormat::TarBz2, bz2_bytes).await.unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].0, "file.txt"); + assert_eq!(entries[0].1, b"bzip2 content"); + } + + #[tokio::test] + async fn test_extract_invalid_tar_returns_error() { + // Fewer than 512 bytes → parser cannot read a complete tar header block. + let bad_bytes = b"this is not a valid tar archive".to_vec(); + let result = extract_tar_entries(ArchiveFormat::Tar, bad_bytes).await; + assert!(result.is_err(), "expected error for invalid tar data"); + } + + #[tokio::test] + async fn test_extract_empty_tar() { + let tar_bytes = make_tar(&[], &[]).await; + let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap(); + assert!(entries.is_empty()); + } + } } diff --git a/crates/protocols/src/swift/encryption.rs b/crates/protocols/src/swift/encryption.rs index 3916fd352..d3051db67 100644 --- a/crates/protocols/src/swift/encryption.rs +++ b/crates/protocols/src/swift/encryption.rs @@ -260,7 +260,7 @@ impl EncryptionMetadata { } /// Check if object should be encrypted based on configuration and headers -pub fn should_encrypt(config: &EncryptionConfig, headers: &axum::http::HeaderMap) -> bool { +pub fn should_encrypt(config: &EncryptionConfig, headers: &http::HeaderMap) -> bool { // Check if encryption is globally enabled if !config.enabled { return false; @@ -447,7 +447,7 @@ mod tests { let key = vec![0u8; 32]; let config = EncryptionConfig::new(true, "test".to_string(), key).unwrap(); - let headers = axum::http::HeaderMap::new(); + let headers = http::HeaderMap::new(); assert!(should_encrypt(&config, &headers)); // Test with disabled config diff --git a/crates/protocols/src/swift/expiration.rs b/crates/protocols/src/swift/expiration.rs index c1740642c..d5a3d150f 100644 --- a/crates/protocols/src/swift/expiration.rs +++ b/crates/protocols/src/swift/expiration.rs @@ -86,7 +86,7 @@ pub fn parse_delete_after(value: &str) -> SwiftResult { /// /// Checks both X-Delete-At and X-Delete-After headers. /// X-Delete-After takes precedence and is converted to X-Delete-At. -pub fn extract_expiration(headers: &axum::http::HeaderMap) -> SwiftResult> { +pub fn extract_expiration(headers: &http::HeaderMap) -> SwiftResult> { // Check X-Delete-After first (takes precedence) if let Some(delete_after) = headers.get("x-delete-after") && let Ok(value_str) = delete_after.to_str() @@ -239,7 +239,7 @@ mod tests { #[test] fn test_extract_expiration_delete_at() { - let mut headers = axum::http::HeaderMap::new(); + let mut headers = http::HeaderMap::new(); headers.insert("x-delete-at", "1740000000".parse().unwrap()); let result = extract_expiration(&headers); @@ -249,7 +249,7 @@ mod tests { #[test] fn test_extract_expiration_delete_after() { - let mut headers = axum::http::HeaderMap::new(); + let mut headers = http::HeaderMap::new(); headers.insert("x-delete-after", "3600".parse().unwrap()); let result = extract_expiration(&headers); @@ -263,7 +263,7 @@ mod tests { #[test] fn test_extract_expiration_delete_after_precedence() { - let mut headers = axum::http::HeaderMap::new(); + let mut headers = http::HeaderMap::new(); headers.insert("x-delete-at", "1740000000".parse().unwrap()); headers.insert("x-delete-after", "3600".parse().unwrap()); @@ -280,7 +280,7 @@ mod tests { #[test] fn test_extract_expiration_none() { - let headers = axum::http::HeaderMap::new(); + let headers = http::HeaderMap::new(); let result = extract_expiration(&headers); assert!(result.is_ok()); diff --git a/crates/protocols/src/swift/formpost.rs b/crates/protocols/src/swift/formpost.rs index 4c7f4a122..9c2c48846 100644 --- a/crates/protocols/src/swift/formpost.rs +++ b/crates/protocols/src/swift/formpost.rs @@ -364,7 +364,7 @@ pub async fn handle_formpost( body: Vec, tempurl_key: &str, credentials: &rustfs_credentials::Credentials, -) -> SwiftResult> { +) -> SwiftResult> { use axum::http::{Response, StatusCode}; // Parse multipart boundary @@ -424,9 +424,9 @@ pub async fn handle_formpost( let reader = std::io::Cursor::new(file.contents.clone()); // Create headers for upload - let mut upload_headers = axum::http::HeaderMap::new(); + let mut upload_headers = http::HeaderMap::new(); if let Some(ct) = &file.content_type - && let Ok(header_value) = axum::http::HeaderValue::from_str(ct) + && let Ok(header_value) = http::HeaderValue::from_str(ct) { upload_headers.insert("content-type", header_value); } diff --git a/crates/protocols/src/swift/handler.rs b/crates/protocols/src/swift/handler.rs index 0c5e5073f..3195c18cb 100644 --- a/crates/protocols/src/swift/handler.rs +++ b/crates/protocols/src/swift/handler.rs @@ -132,7 +132,7 @@ async fn handle_swift_request( // Get account TempURL key let tempurl_key = super::account::get_tempurl_key(account, &credentials).await?; - if let Some(key) = tempurl_key { + return if let Some(key) = tempurl_key { // Validate TempURL signature let tempurl = tempurl::TempURL::new(key); let path = uri.path(); @@ -144,11 +144,11 @@ async fn handle_swift_request( // Reconstruct request for object operation let req = Request::from_parts(parts, body); - return handle_tempurl_object_request(req, route).await; + handle_tempurl_object_request(req, route).await } else { // No TempURL key configured for this account - return Err(SwiftError::Unauthorized("TempURL key not configured for this account".to_string())); - } + Err(SwiftError::Unauthorized("TempURL key not configured for this account".to_string())) + }; } // No TempURL or TempURL validation failed - require normal authentication @@ -476,7 +476,7 @@ async fn handle_authenticated_request( // FormPost upload - get TempURL key for signature validation let tempurl_key = super::account::get_tempurl_key(&account, &Some(credentials.clone())).await?; - if let Some(key) = tempurl_key { + return if let Some(key) = tempurl_key { // Collect body for multipart parsing use http_body_util::BodyExt; let body_bytes = body @@ -489,19 +489,11 @@ async fn handle_authenticated_request( // Build path for signature validation let path = format!("/v1/{}/{}", account, container); - return super::formpost::handle_formpost( - &account, - &container, - &path, - ct_str, - body_bytes, - &key, - &credentials, - ) - .await; + super::formpost::handle_formpost(&account, &container, &path, ct_str, body_bytes, &key, &credentials) + .await } else { - return Err(SwiftError::Unauthorized("TempURL key not configured for FormPost".to_string())); - } + Err(SwiftError::Unauthorized("TempURL key not configured for FormPost".to_string())) + }; } // Check for versioning headers first @@ -1018,9 +1010,8 @@ async fn handle_authenticated_request( } // Type alias for complex symlink resolution future -type SymlinkResolutionFuture<'a> = std::pin::Pin< - Box), SwiftError>> + Send + 'a>, ->; +type SymlinkResolutionFuture<'a> = + Pin), SwiftError>> + Send + 'a>>; /// Resolve symlink chain recursively /// @@ -1073,7 +1064,7 @@ async fn handle_object_get( account: &str, container: &str, object: &str, - headers: &axum::http::HeaderMap, + headers: &http::HeaderMap, credentials: &Option, ) -> Result, SwiftError> { // For TempURL requests, credentials will be None @@ -1269,7 +1260,7 @@ async fn handle_object_put( container: &str, object: &str, body: Body, - headers: &axum::http::HeaderMap, + headers: &http::HeaderMap, credentials: &Option, ) -> Result, SwiftError> { let creds = credentials @@ -1323,7 +1314,7 @@ async fn check_container_acl( container: &str, credentials: &Credentials, is_write: bool, - headers: &axum::http::HeaderMap, + headers: &http::HeaderMap, ) -> Result<(), SwiftError> { // Get container ACLs let acl = container::get_container_acl(account, container, credentials).await?; @@ -1396,7 +1387,7 @@ async fn inject_cors_headers( container: Option<&str>, account: &str, credentials: &Credentials, - request_headers: &axum::http::HeaderMap, + request_headers: &http::HeaderMap, ) -> Response { // Only inject CORS for container/object routes if let Some(container_name) = container { diff --git a/crates/protocols/src/swift/symlink.rs b/crates/protocols/src/swift/symlink.rs index b2eabfcbb..baf6f9f70 100644 --- a/crates/protocols/src/swift/symlink.rs +++ b/crates/protocols/src/swift/symlink.rs @@ -128,7 +128,7 @@ impl SymlinkTarget { } /// Extract symlink target from request headers -pub fn extract_symlink_target(headers: &axum::http::HeaderMap) -> SwiftResult> { +pub fn extract_symlink_target(headers: &http::HeaderMap) -> SwiftResult> { if let Some(target_header) = headers.get("x-object-symlink-target") { let target_str = target_header .to_str() @@ -248,7 +248,7 @@ mod tests { #[test] fn test_extract_symlink_target_present() { - let mut headers = axum::http::HeaderMap::new(); + let mut headers = http::HeaderMap::new(); headers.insert("x-object-symlink-target", "target.txt".parse().unwrap()); let result = extract_symlink_target(&headers).unwrap(); @@ -261,7 +261,7 @@ mod tests { #[test] fn test_extract_symlink_target_absent() { - let headers = axum::http::HeaderMap::new(); + let headers = http::HeaderMap::new(); let result = extract_symlink_target(&headers).unwrap(); assert!(result.is_none()); } diff --git a/crates/protocols/src/swift/tempurl.rs b/crates/protocols/src/swift/tempurl.rs index 6e1b030be..c9a0f48a5 100644 --- a/crates/protocols/src/swift/tempurl.rs +++ b/crates/protocols/src/swift/tempurl.rs @@ -1,3 +1,17 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! TempURL (Temporary URL) support for OpenStack Swift //! //! TempURLs provide time-limited access to objects without requiring authentication. diff --git a/crates/protocols/src/swift/versioning.rs b/crates/protocols/src/swift/versioning.rs index c950cf440..615f5d9ef 100644 --- a/crates/protocols/src/swift/versioning.rs +++ b/crates/protocols/src/swift/versioning.rs @@ -1,3 +1,17 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! Object Versioning Support for Swift API //! //! Implements Swift object versioning where old versions are automatically