mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
refactor(protocols): replace tar with astral-tokio-tar for async processing (#2099)
Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
Generated
+40
-75
@@ -613,9 +613,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
||||
|
||||
[[package]]
|
||||
name = "aws-config"
|
||||
version = "1.8.14"
|
||||
version = "1.8.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2"
|
||||
checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
@@ -633,7 +633,7 @@ dependencies = [
|
||||
"fastrand",
|
||||
"hex",
|
||||
"http 1.4.0",
|
||||
"ring",
|
||||
"sha1 0.10.6",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -643,9 +643,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-credential-types"
|
||||
version = "1.2.13"
|
||||
version = "1.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d203b0bf2626dcba8665f5cd0871d7c2c0930223d6b6be9097592fea21242d0"
|
||||
checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -678,9 +678,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-runtime"
|
||||
version = "1.7.1"
|
||||
version = "1.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ede2ddc593e6c8acc6ce3358c28d6677a6dc49b65ba4b37a2befe14a11297e75"
|
||||
checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-sigv4",
|
||||
@@ -706,9 +706,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.124.0"
|
||||
version = "1.125.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "744c09d75dfec039a05cf8e117c995ded3b0baffa6eb83f3ed7075a01d8d8947"
|
||||
checksum = "223f5c95650d9557925a91f4c2db3def189e8f659452134a29e5cd2d37d708ed"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
@@ -741,9 +741,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sso"
|
||||
version = "1.95.0"
|
||||
version = "1.96.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00c5ff27c6ba2cbd95e6e26e2e736676fdf6bcf96495b187733f521cfe4ce448"
|
||||
checksum = "f64a6eded248c6b453966e915d32aeddb48ea63ad17932682774eb026fbef5b1"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
@@ -765,9 +765,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-ssooidc"
|
||||
version = "1.97.0"
|
||||
version = "1.98.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d186f1e5a3694a188e5a0640b3115ccc6e084d104e16fd6ba968dca072ffef8"
|
||||
checksum = "db96d720d3c622fcbe08bae1c4b04a72ce6257d8b0584cb5418da00ae20a344f"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
@@ -789,9 +789,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sts"
|
||||
version = "1.99.0"
|
||||
version = "1.100.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9acba7c62f3d4e2408fa998a3a8caacd8b9a5b5549cf36e2372fbdae329d5449"
|
||||
checksum = "fafbdda43b93f57f699c5dfe8328db590b967b8a820a13ccdd6687355dfcc7ca"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
@@ -814,9 +814,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "1.4.1"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37411f8e0f4bea0c3ca0958ce7f18f6439db24d555dbd809787262cd00926aa9"
|
||||
checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
@@ -1036,9 +1036,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-types"
|
||||
version = "1.3.13"
|
||||
version = "1.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0470cc047657c6e286346bdf10a8719d26efd6a91626992e0e64481e44323e96"
|
||||
checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-async",
|
||||
@@ -1311,16 +1311,6 @@ dependencies = [
|
||||
"bytes",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bzip2"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8"
|
||||
dependencies = [
|
||||
"bzip2-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bzip2"
|
||||
version = "0.6.1"
|
||||
@@ -1330,16 +1320,6 @@ dependencies = [
|
||||
"libbz2-rs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bzip2-sys"
|
||||
version = "0.1.13+1.0.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "camino"
|
||||
version = "1.2.2"
|
||||
@@ -1490,7 +1470,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -1604,7 +1584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7"
|
||||
dependencies = [
|
||||
"brotli",
|
||||
"bzip2 0.6.1",
|
||||
"bzip2",
|
||||
"compression-core",
|
||||
"flate2",
|
||||
"liblzma",
|
||||
@@ -1961,9 +1941,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
@@ -2232,7 +2212,7 @@ dependencies = [
|
||||
"arrow-schema",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"bzip2 0.6.1",
|
||||
"bzip2",
|
||||
"chrono",
|
||||
"datafusion-catalog",
|
||||
"datafusion-catalog-listing",
|
||||
@@ -2370,7 +2350,7 @@ dependencies = [
|
||||
"async-compression",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"bzip2 0.6.1",
|
||||
"bzip2",
|
||||
"chrono",
|
||||
"datafusion-common",
|
||||
"datafusion-common-runtime",
|
||||
@@ -3045,7 +3025,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3626,9 +3606,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.9"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -6225,9 +6205,9 @@ checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
|
||||
|
||||
[[package]]
|
||||
name = "pprof-pyroscope-fork"
|
||||
version = "0.1500.2"
|
||||
version = "0.1500.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fa86bc766c4e3e07526f21028ad9cdd112903c2cc646b36724c54978fb42f81"
|
||||
checksum = "79f090659c6f3f8c12785dabca0cb7f40154a40dc254c06848598ff31c3be1d8"
|
||||
dependencies = [
|
||||
"aligned-vec",
|
||||
"backtrace",
|
||||
@@ -7748,12 +7728,12 @@ dependencies = [
|
||||
name = "rustfs-protocols"
|
||||
version = "0.0.5"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"bzip2 0.4.4",
|
||||
"flate2",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"hex",
|
||||
@@ -7761,13 +7741,10 @@ dependencies = [
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"libunftp",
|
||||
"md5",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"quick-xml 0.39.2",
|
||||
"rand 0.10.0",
|
||||
"regex",
|
||||
"rustfs-credentials",
|
||||
"rustfs-ecstore",
|
||||
@@ -7782,7 +7759,6 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha1 0.11.0-rc.5",
|
||||
"sha2 0.11.0-rc.5",
|
||||
"tar",
|
||||
"thiserror 2.0.18",
|
||||
"time",
|
||||
"tokio",
|
||||
@@ -8213,7 +8189,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.14.0-dev"
|
||||
source = "git+https://github.com/s3s-project/s3s?rev=2fa67f48764a0d80f304304d34c3f978e0fdacf5#2fa67f48764a0d80f304304d34c3f978e0fdacf5"
|
||||
source = "git+https://github.com/s3s-project/s3s?rev=4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990#4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -8831,12 +8807,12 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.6.2"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9142,17 +9118,6 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
||||
|
||||
[[package]]
|
||||
name = "tar"
|
||||
version = "0.4.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
|
||||
dependencies = [
|
||||
"filetime",
|
||||
"libc",
|
||||
"xattr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "temp-env"
|
||||
version = "0.3.6"
|
||||
@@ -9852,13 +9817,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.21.0"
|
||||
version = "1.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb"
|
||||
checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
|
||||
dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"js-sys",
|
||||
"rand 0.9.2",
|
||||
"rand 0.10.0",
|
||||
"serde_core",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
@@ -10777,7 +10742,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b680f2a0cd479b4cff6e1233c483fdead418106eae419dc60200ae9850f6d004"
|
||||
dependencies = [
|
||||
"aes 0.8.4",
|
||||
"bzip2 0.6.1",
|
||||
"bzip2",
|
||||
"constant_time_eq",
|
||||
"crc32fast",
|
||||
"deflate64",
|
||||
|
||||
+7
-7
@@ -122,7 +122,7 @@ http = "1.4.0"
|
||||
http-body = "1.0.1"
|
||||
http-body-util = "0.1.3"
|
||||
reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
|
||||
socket2 = { version = "0.6.2", features = ["all"] }
|
||||
socket2 = { version = "0.6.3", features = ["all"] }
|
||||
tokio = { version = "1.50.0", features = ["fs", "rt-multi-thread"] }
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
|
||||
tokio-stream = { version = "0.1.18" }
|
||||
@@ -180,9 +180,9 @@ arc-swap = "1.8.2"
|
||||
astral-tokio-tar = "0.5.6"
|
||||
atoi = "2.0.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.8.14" }
|
||||
aws-credential-types = { version = "1.2.13" }
|
||||
aws-sdk-s3 = { version = "1.124.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-config = { version = "1.8.15" }
|
||||
aws-credential-types = { version = "1.2.14" }
|
||||
aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
|
||||
aws-smithy-types = { version = "1.4.6" }
|
||||
backtrace = "0.3.76"
|
||||
@@ -237,7 +237,7 @@ rumqttc = { version = "0.25.1" }
|
||||
rustix = { version = "1.1.4", features = ["fs"] }
|
||||
rust-embed = { version = "8.11.0" }
|
||||
rustc-hash = { version = "2.1.1" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "2fa67f48764a0d80f304304d34c3f978e0fdacf5", features = ["minio"] }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990", features = ["minio"] }
|
||||
serial_test = "3.4.0"
|
||||
shadow-rs = { version = "1.7.0", default-features = false }
|
||||
siphasher = "1.0.2"
|
||||
@@ -260,7 +260,7 @@ tracing-subscriber = { version = "0.3.22", features = ["env-filter", "time"] }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.21.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
uuid = { version = "1.22.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
vaultrs = { version = "0.7.4" }
|
||||
walkdir = "2.5.0"
|
||||
wildmatch = { version = "2.6.1", features = ["serde"] }
|
||||
@@ -296,7 +296,7 @@ jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
|
||||
# Used to generate CPU performance analysis data and flame diagrams
|
||||
# pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] }
|
||||
# Pyroscope uses a patched pprof, until they merge back upstream, replace all references. Otherwise, two pprof libs with symbol collision.
|
||||
pprof = { package = "pprof-pyroscope-fork", version = "0.1500.2", features = ["flamegraph", "protobuf-codec"] }
|
||||
pprof = { package = "pprof-pyroscope-fork", version = "0.1500.3", features = ["flamegraph", "protobuf-codec"] }
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
ignored = ["rustfs", "rustfs-mcp"]
|
||||
|
||||
@@ -34,13 +34,11 @@ swift = [
|
||||
"dep:rustfs-rio",
|
||||
"dep:axum",
|
||||
"dep:http",
|
||||
"dep:hyper",
|
||||
"dep:tower",
|
||||
"dep:regex",
|
||||
"dep:percent-encoding",
|
||||
"dep:sha2",
|
||||
"dep:uuid",
|
||||
"dep:pin-project-lite",
|
||||
"dep:futures",
|
||||
"dep:http-body",
|
||||
"dep:http-body-util",
|
||||
@@ -52,11 +50,9 @@ swift = [
|
||||
"dep:hmac",
|
||||
"dep:sha1",
|
||||
"dep:hex",
|
||||
"dep:tar",
|
||||
"dep:flate2",
|
||||
"dep:bzip2",
|
||||
"dep:astral-tokio-tar",
|
||||
"dep:base64",
|
||||
"dep:rand",
|
||||
"dep:async-compression",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
@@ -96,13 +92,11 @@ rustfs-ecstore = { workspace = true, optional = true }
|
||||
rustfs-rio = { workspace = true, optional = true }
|
||||
axum = { workspace = true, optional = true }
|
||||
http = { workspace = true, optional = true }
|
||||
hyper = { workspace = true, optional = true }
|
||||
tower = { workspace = true, optional = true }
|
||||
regex = { workspace = true, optional = true }
|
||||
percent-encoding = { workspace = true, optional = true }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
uuid = { workspace = true, optional = true }
|
||||
pin-project-lite = { workspace = true, optional = true }
|
||||
futures = { workspace = true, optional = true }
|
||||
http-body = { workspace = true, optional = true }
|
||||
http-body-util = { workspace = true, optional = true }
|
||||
@@ -114,11 +108,9 @@ quick-xml = { workspace = true, optional = true, features = ["serialize"] }
|
||||
hmac = { workspace = true, optional = true }
|
||||
sha1 = { workspace = true, optional = true }
|
||||
hex = { workspace = true, optional = true }
|
||||
tar = { version = "0.4", optional = true }
|
||||
flate2 = { workspace = true, optional = true }
|
||||
bzip2 = { version = "0.4", optional = true }
|
||||
astral-tokio-tar = { workspace = true, optional = true }
|
||||
base64 = { workspace = true, optional = true }
|
||||
rand = { workspace = true, optional = true }
|
||||
async-compression = { workspace = true, optional = true, features = ["tokio", "gzip", "bzip2"] }
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
all-features = true
|
||||
|
||||
@@ -76,10 +76,10 @@
|
||||
|
||||
use super::{SwiftError, SwiftResult, container, object};
|
||||
use axum::http::{Response, StatusCode};
|
||||
use futures::StreamExt;
|
||||
use rustfs_credentials::Credentials;
|
||||
use s3s::Body;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::io::Read;
|
||||
use tracing::{debug, error};
|
||||
|
||||
/// Result of a single delete operation
|
||||
@@ -340,8 +340,8 @@ pub async fn handle_bulk_extract(
|
||||
return Err(SwiftError::NotFound(format!("Container not found: {}", container)));
|
||||
}
|
||||
|
||||
// Parse archive and collect entries (without holding the archive)
|
||||
let entries = extract_tar_entries(format, body)?;
|
||||
// Parse archive and collect all entries into memory (entire archive and file contents are buffered)
|
||||
let entries = extract_tar_entries(format, body).await?;
|
||||
|
||||
// Now upload each entry (async operations)
|
||||
for (path_str, contents) in entries {
|
||||
@@ -352,7 +352,7 @@ pub async fn handle_bulk_extract(
|
||||
&path_str,
|
||||
credentials,
|
||||
std::io::Cursor::new(contents),
|
||||
&axum::http::HeaderMap::new(),
|
||||
&http::HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -395,30 +395,30 @@ pub async fn handle_bulk_extract(
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to build response: {}", e)))
|
||||
}
|
||||
|
||||
/// Extract tar entries synchronously to avoid Send issues
|
||||
fn extract_tar_entries(format: ArchiveFormat, body: Vec<u8>) -> SwiftResult<Vec<(String, Vec<u8>)>> {
|
||||
/// Extract tar entries using async I/O and return them as in-memory buffers
|
||||
async fn extract_tar_entries(format: ArchiveFormat, body: Vec<u8>) -> SwiftResult<Vec<(String, Vec<u8>)>> {
|
||||
// Create appropriate reader based on format
|
||||
let reader: Box<dyn Read> = match format {
|
||||
let reader: Box<dyn tokio::io::AsyncRead + Unpin + Send> = match format {
|
||||
ArchiveFormat::Tar => Box::new(std::io::Cursor::new(body)),
|
||||
ArchiveFormat::TarGz => {
|
||||
let cursor = std::io::Cursor::new(body);
|
||||
Box::new(flate2::read::GzDecoder::new(cursor))
|
||||
Box::new(async_compression::tokio::bufread::GzipDecoder::new(tokio::io::BufReader::new(cursor)))
|
||||
}
|
||||
ArchiveFormat::TarBz2 => {
|
||||
let cursor = std::io::Cursor::new(body);
|
||||
Box::new(bzip2::read::BzDecoder::new(cursor))
|
||||
Box::new(async_compression::tokio::bufread::BzDecoder::new(tokio::io::BufReader::new(cursor)))
|
||||
}
|
||||
};
|
||||
|
||||
// Parse tar archive
|
||||
let mut archive = tar::Archive::new(reader);
|
||||
let mut archive = tokio_tar::Archive::new(reader);
|
||||
let mut entries = Vec::new();
|
||||
let mut entries_iter = archive
|
||||
.entries()
|
||||
.map_err(|e| SwiftError::BadRequest(format!("Failed to read tar archive: {}", e)))?;
|
||||
|
||||
// Extract each entry
|
||||
for entry in archive
|
||||
.entries()
|
||||
.map_err(|e| SwiftError::BadRequest(format!("Failed to read tar archive: {}", e)))?
|
||||
{
|
||||
while let Some(entry) = entries_iter.next().await {
|
||||
let mut entry = entry.map_err(|e| SwiftError::BadRequest(format!("Failed to read tar entry: {}", e)))?;
|
||||
|
||||
// Get entry path
|
||||
@@ -436,7 +436,7 @@ fn extract_tar_entries(format: ArchiveFormat, body: Vec<u8>) -> SwiftResult<Vec<
|
||||
|
||||
// Read file contents
|
||||
let mut contents = Vec::new();
|
||||
if let Err(e) = entry.read_to_end(&mut contents) {
|
||||
if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut entry, &mut contents).await {
|
||||
error!("Failed to read tar entry {}: {}", path_str, e);
|
||||
continue;
|
||||
}
|
||||
@@ -552,4 +552,117 @@ mod tests {
|
||||
|
||||
assert_eq!(paths.len(), 3);
|
||||
}
|
||||
|
||||
/// Tests for the `extract_tar_entries` async function.
|
||||
///
|
||||
/// Conditionally compiled with the `swift` feature, which gates the
|
||||
/// `tokio_tar` (astral-tokio-tar) and `async_compression` dependencies.
|
||||
#[cfg(feature = "swift")]
|
||||
mod tar_extraction {
|
||||
use super::*;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
/// Builds an uncompressed tar archive in memory.
|
||||
///
|
||||
/// * `files` – `(path, content)` pairs added as regular files.
|
||||
/// * `dirs` – paths added as directory entries.
|
||||
async fn make_tar(files: &[(&str, &[u8])], dirs: &[&str]) -> Vec<u8> {
|
||||
let buf = std::io::Cursor::new(Vec::new());
|
||||
let mut builder = tokio_tar::Builder::new(buf);
|
||||
|
||||
for &dir in dirs {
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_entry_type(tokio_tar::EntryType::Directory);
|
||||
header.set_size(0);
|
||||
header.set_mode(0o755);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, dir, std::io::Cursor::new(&[] as &[u8]))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for &(name, data) in files {
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(data.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, name, std::io::Cursor::new(data))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
builder.into_inner().await.unwrap().into_inner()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_extract_plain_tar_paths_and_contents() {
|
||||
let tar_bytes = make_tar(&[("file1.txt", b"hello"), ("dir/file2.txt", b"world")], &[]).await;
|
||||
let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap();
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert_eq!(entries[0].0, "file1.txt");
|
||||
assert_eq!(entries[0].1, b"hello");
|
||||
assert_eq!(entries[1].0, "dir/file2.txt");
|
||||
assert_eq!(entries[1].1, b"world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_extract_tar_skips_directories() {
|
||||
let tar_bytes = make_tar(&[("file.txt", b"content")], &["subdir/", "another/"]).await;
|
||||
let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap();
|
||||
// Directory entries must be filtered out; only the regular file is returned.
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].0, "file.txt");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_extract_tar_gz() {
|
||||
let tar_bytes = make_tar(&[("file.txt", b"compressed content")], &[]).await;
|
||||
|
||||
// Compress with gzip.
|
||||
let cursor = std::io::Cursor::new(Vec::new());
|
||||
let mut encoder = async_compression::tokio::write::GzipEncoder::new(cursor);
|
||||
encoder.write_all(&tar_bytes).await.unwrap();
|
||||
encoder.shutdown().await.unwrap();
|
||||
let gz_bytes = encoder.into_inner().into_inner();
|
||||
|
||||
let entries = extract_tar_entries(ArchiveFormat::TarGz, gz_bytes).await.unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].0, "file.txt");
|
||||
assert_eq!(entries[0].1, b"compressed content");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_extract_tar_bz2() {
|
||||
let tar_bytes = make_tar(&[("file.txt", b"bzip2 content")], &[]).await;
|
||||
|
||||
// Compress with bzip2.
|
||||
let cursor = std::io::Cursor::new(Vec::new());
|
||||
let mut encoder = async_compression::tokio::write::BzEncoder::new(cursor);
|
||||
encoder.write_all(&tar_bytes).await.unwrap();
|
||||
encoder.shutdown().await.unwrap();
|
||||
let bz2_bytes = encoder.into_inner().into_inner();
|
||||
|
||||
let entries = extract_tar_entries(ArchiveFormat::TarBz2, bz2_bytes).await.unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].0, "file.txt");
|
||||
assert_eq!(entries[0].1, b"bzip2 content");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_extract_invalid_tar_returns_error() {
|
||||
// Fewer than 512 bytes → parser cannot read a complete tar header block.
|
||||
let bad_bytes = b"this is not a valid tar archive".to_vec();
|
||||
let result = extract_tar_entries(ArchiveFormat::Tar, bad_bytes).await;
|
||||
assert!(result.is_err(), "expected error for invalid tar data");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_extract_empty_tar() {
|
||||
let tar_bytes = make_tar(&[], &[]).await;
|
||||
let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap();
|
||||
assert!(entries.is_empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -260,7 +260,7 @@ impl EncryptionMetadata {
|
||||
}
|
||||
|
||||
/// Check if object should be encrypted based on configuration and headers
|
||||
pub fn should_encrypt(config: &EncryptionConfig, headers: &axum::http::HeaderMap) -> bool {
|
||||
pub fn should_encrypt(config: &EncryptionConfig, headers: &http::HeaderMap) -> bool {
|
||||
// Check if encryption is globally enabled
|
||||
if !config.enabled {
|
||||
return false;
|
||||
@@ -447,7 +447,7 @@ mod tests {
|
||||
let key = vec![0u8; 32];
|
||||
let config = EncryptionConfig::new(true, "test".to_string(), key).unwrap();
|
||||
|
||||
let headers = axum::http::HeaderMap::new();
|
||||
let headers = http::HeaderMap::new();
|
||||
assert!(should_encrypt(&config, &headers));
|
||||
|
||||
// Test with disabled config
|
||||
|
||||
@@ -86,7 +86,7 @@ pub fn parse_delete_after(value: &str) -> SwiftResult<u64> {
|
||||
///
|
||||
/// Checks both X-Delete-At and X-Delete-After headers.
|
||||
/// X-Delete-After takes precedence and is converted to X-Delete-At.
|
||||
pub fn extract_expiration(headers: &axum::http::HeaderMap) -> SwiftResult<Option<u64>> {
|
||||
pub fn extract_expiration(headers: &http::HeaderMap) -> SwiftResult<Option<u64>> {
|
||||
// Check X-Delete-After first (takes precedence)
|
||||
if let Some(delete_after) = headers.get("x-delete-after")
|
||||
&& let Ok(value_str) = delete_after.to_str()
|
||||
@@ -239,7 +239,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_expiration_delete_at() {
|
||||
let mut headers = axum::http::HeaderMap::new();
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("x-delete-at", "1740000000".parse().unwrap());
|
||||
|
||||
let result = extract_expiration(&headers);
|
||||
@@ -249,7 +249,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_expiration_delete_after() {
|
||||
let mut headers = axum::http::HeaderMap::new();
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("x-delete-after", "3600".parse().unwrap());
|
||||
|
||||
let result = extract_expiration(&headers);
|
||||
@@ -263,7 +263,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_expiration_delete_after_precedence() {
|
||||
let mut headers = axum::http::HeaderMap::new();
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("x-delete-at", "1740000000".parse().unwrap());
|
||||
headers.insert("x-delete-after", "3600".parse().unwrap());
|
||||
|
||||
@@ -280,7 +280,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_expiration_none() {
|
||||
let headers = axum::http::HeaderMap::new();
|
||||
let headers = http::HeaderMap::new();
|
||||
|
||||
let result = extract_expiration(&headers);
|
||||
assert!(result.is_ok());
|
||||
|
||||
@@ -364,7 +364,7 @@ pub async fn handle_formpost(
|
||||
body: Vec<u8>,
|
||||
tempurl_key: &str,
|
||||
credentials: &rustfs_credentials::Credentials,
|
||||
) -> SwiftResult<axum::http::Response<s3s::Body>> {
|
||||
) -> SwiftResult<http::Response<s3s::Body>> {
|
||||
use axum::http::{Response, StatusCode};
|
||||
|
||||
// Parse multipart boundary
|
||||
@@ -424,9 +424,9 @@ pub async fn handle_formpost(
|
||||
let reader = std::io::Cursor::new(file.contents.clone());
|
||||
|
||||
// Create headers for upload
|
||||
let mut upload_headers = axum::http::HeaderMap::new();
|
||||
let mut upload_headers = http::HeaderMap::new();
|
||||
if let Some(ct) = &file.content_type
|
||||
&& let Ok(header_value) = axum::http::HeaderValue::from_str(ct)
|
||||
&& let Ok(header_value) = http::HeaderValue::from_str(ct)
|
||||
{
|
||||
upload_headers.insert("content-type", header_value);
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ async fn handle_swift_request(
|
||||
// Get account TempURL key
|
||||
let tempurl_key = super::account::get_tempurl_key(account, &credentials).await?;
|
||||
|
||||
if let Some(key) = tempurl_key {
|
||||
return if let Some(key) = tempurl_key {
|
||||
// Validate TempURL signature
|
||||
let tempurl = tempurl::TempURL::new(key);
|
||||
let path = uri.path();
|
||||
@@ -144,11 +144,11 @@ async fn handle_swift_request(
|
||||
|
||||
// Reconstruct request for object operation
|
||||
let req = Request::from_parts(parts, body);
|
||||
return handle_tempurl_object_request(req, route).await;
|
||||
handle_tempurl_object_request(req, route).await
|
||||
} else {
|
||||
// No TempURL key configured for this account
|
||||
return Err(SwiftError::Unauthorized("TempURL key not configured for this account".to_string()));
|
||||
}
|
||||
Err(SwiftError::Unauthorized("TempURL key not configured for this account".to_string()))
|
||||
};
|
||||
}
|
||||
|
||||
// No TempURL or TempURL validation failed - require normal authentication
|
||||
@@ -476,7 +476,7 @@ async fn handle_authenticated_request(
|
||||
// FormPost upload - get TempURL key for signature validation
|
||||
let tempurl_key = super::account::get_tempurl_key(&account, &Some(credentials.clone())).await?;
|
||||
|
||||
if let Some(key) = tempurl_key {
|
||||
return if let Some(key) = tempurl_key {
|
||||
// Collect body for multipart parsing
|
||||
use http_body_util::BodyExt;
|
||||
let body_bytes = body
|
||||
@@ -489,19 +489,11 @@ async fn handle_authenticated_request(
|
||||
// Build path for signature validation
|
||||
let path = format!("/v1/{}/{}", account, container);
|
||||
|
||||
return super::formpost::handle_formpost(
|
||||
&account,
|
||||
&container,
|
||||
&path,
|
||||
ct_str,
|
||||
body_bytes,
|
||||
&key,
|
||||
&credentials,
|
||||
)
|
||||
.await;
|
||||
super::formpost::handle_formpost(&account, &container, &path, ct_str, body_bytes, &key, &credentials)
|
||||
.await
|
||||
} else {
|
||||
return Err(SwiftError::Unauthorized("TempURL key not configured for FormPost".to_string()));
|
||||
}
|
||||
Err(SwiftError::Unauthorized("TempURL key not configured for FormPost".to_string()))
|
||||
};
|
||||
}
|
||||
|
||||
// Check for versioning headers first
|
||||
@@ -1018,9 +1010,8 @@ async fn handle_authenticated_request(
|
||||
}
|
||||
|
||||
// Type alias for complex symlink resolution future
|
||||
type SymlinkResolutionFuture<'a> = std::pin::Pin<
|
||||
Box<dyn std::future::Future<Output = Result<(String, String, String, Option<String>), SwiftError>> + Send + 'a>,
|
||||
>;
|
||||
type SymlinkResolutionFuture<'a> =
|
||||
Pin<Box<dyn std::future::Future<Output = Result<(String, String, String, Option<String>), SwiftError>> + Send + 'a>>;
|
||||
|
||||
/// Resolve symlink chain recursively
|
||||
///
|
||||
@@ -1073,7 +1064,7 @@ async fn handle_object_get(
|
||||
account: &str,
|
||||
container: &str,
|
||||
object: &str,
|
||||
headers: &axum::http::HeaderMap,
|
||||
headers: &http::HeaderMap,
|
||||
credentials: &Option<Credentials>,
|
||||
) -> Result<Response<Body>, SwiftError> {
|
||||
// For TempURL requests, credentials will be None
|
||||
@@ -1269,7 +1260,7 @@ async fn handle_object_put(
|
||||
container: &str,
|
||||
object: &str,
|
||||
body: Body,
|
||||
headers: &axum::http::HeaderMap,
|
||||
headers: &http::HeaderMap,
|
||||
credentials: &Option<Credentials>,
|
||||
) -> Result<Response<Body>, SwiftError> {
|
||||
let creds = credentials
|
||||
@@ -1323,7 +1314,7 @@ async fn check_container_acl(
|
||||
container: &str,
|
||||
credentials: &Credentials,
|
||||
is_write: bool,
|
||||
headers: &axum::http::HeaderMap,
|
||||
headers: &http::HeaderMap,
|
||||
) -> Result<(), SwiftError> {
|
||||
// Get container ACLs
|
||||
let acl = container::get_container_acl(account, container, credentials).await?;
|
||||
@@ -1396,7 +1387,7 @@ async fn inject_cors_headers(
|
||||
container: Option<&str>,
|
||||
account: &str,
|
||||
credentials: &Credentials,
|
||||
request_headers: &axum::http::HeaderMap,
|
||||
request_headers: &http::HeaderMap,
|
||||
) -> Response<Body> {
|
||||
// Only inject CORS for container/object routes
|
||||
if let Some(container_name) = container {
|
||||
|
||||
@@ -128,7 +128,7 @@ impl SymlinkTarget {
|
||||
}
|
||||
|
||||
/// Extract symlink target from request headers
|
||||
pub fn extract_symlink_target(headers: &axum::http::HeaderMap) -> SwiftResult<Option<SymlinkTarget>> {
|
||||
pub fn extract_symlink_target(headers: &http::HeaderMap) -> SwiftResult<Option<SymlinkTarget>> {
|
||||
if let Some(target_header) = headers.get("x-object-symlink-target") {
|
||||
let target_str = target_header
|
||||
.to_str()
|
||||
@@ -248,7 +248,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_symlink_target_present() {
|
||||
let mut headers = axum::http::HeaderMap::new();
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("x-object-symlink-target", "target.txt".parse().unwrap());
|
||||
|
||||
let result = extract_symlink_target(&headers).unwrap();
|
||||
@@ -261,7 +261,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_symlink_target_absent() {
|
||||
let headers = axum::http::HeaderMap::new();
|
||||
let headers = http::HeaderMap::new();
|
||||
let result = extract_symlink_target(&headers).unwrap();
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
@@ -1,3 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! TempURL (Temporary URL) support for OpenStack Swift
|
||||
//!
|
||||
//! TempURLs provide time-limited access to objects without requiring authentication.
|
||||
|
||||
@@ -1,3 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Object Versioning Support for Swift API
|
||||
//!
|
||||
//! Implements Swift object versioning where old versions are automatically
|
||||
|
||||
Reference in New Issue
Block a user