From 311d85219e77604f7c0923436d9953a07be275fa Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 5 Sep 2026 08:11:20 +0800 Subject: [PATCH] test(s3): add Snowball tar-codec compatibility fixtures --- .gitignore | 1 + Cargo.lock | 229 +++++++- Cargo.toml | 5 +- crates/zip/Cargo.toml | 9 + .../snowball/minio-go-v7.3.0/README.md | 24 + .../snowball/minio-go-v7.3.0/generate/go.mod | 26 + .../snowball/minio-go-v7.3.0/generate/go.sum | 59 ++ .../snowball/minio-go-v7.3.0/generate/main.go | 193 ++++++ .../snowball/minio-go-v7.3.0/manifest.json | 51 ++ .../snowball/minio-go-v7.3.0/snowball.tar | Bin 0 -> 4096 bytes .../snowball/minio-go-v7.3.0/snowball.tar.s2 | Bin 0 -> 528 bytes crates/zip/tests/snowball_tar_codec_compat.rs | 548 ++++++++++++++++++ deny.toml | 4 +- docs/architecture/compat-cleanup-register.md | 2 +- 14 files changed, 1138 insertions(+), 13 deletions(-) create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2 create mode 100644 crates/zip/tests/snowball_tar_codec_compat.rs diff --git a/.gitignore b/.gitignore index b8d2eecbf..60325653d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ profile.json *.zst .secrets *.go +!crates/zip/tests/fixtures/snowball/**/generate/*.go *.pb *.svg deploy/logs/*.log.* diff --git a/Cargo.lock b/Cargo.lock index e9e7d9fe5..9369b1a5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "amq-protocol" version = "10.6.3" @@ -330,6 +336,19 @@ dependencies = [ "rustversion", ] +[[package]] +name = "archive-trait" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6080ea14ccf9019d7ce572c319e581c20a805c9bdc6dbfb9b988da003cbd1a3" +dependencies = [ + "cap-std", + "thiserror 2.0.20", + "tokio", + "walkdir", + "windows-sys 0.60.2", +] + [[package]] name = "arcstr" version = "1.2.0" @@ -1873,6 +1892,36 @@ dependencies = [ "serde_core", ] +[[package]] +name = "cap-primitives" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 3.0.1", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.61.2", + "winx", +] + +[[package]] +name = "cap-std" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes 3.0.1", + "rustix", +] + [[package]] name = "cargo-platform" version = "0.3.3" @@ -4442,6 +4491,17 @@ dependencies = [ "pe-unwind-info", ] +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes 2.0.4", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -5626,6 +5686,28 @@ dependencies = [ "tempfile", ] +[[package]] +name = "io-extras" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" +dependencies = [ + "io-lifetimes 3.0.1", + "windows-sys 0.52.0", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + +[[package]] +name = "io-lifetimes" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" + [[package]] name = "io-uring" version = "0.7.14" @@ -6328,6 +6410,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "md-5" version = "0.10.6" @@ -10903,9 +10991,16 @@ dependencies = [ name = "rustfs-zip" version = "1.0.0-rc.5" dependencies = [ + "astral-tokio-tar", "async-compression", + "futures", "hotpath", "rustfs-rio", + "serde", + "serde_json", + "sha2 0.11.0", + "tar-codec", + "tar-framing", "thiserror 2.0.20", "tokio", ] @@ -10966,6 +11061,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", +] + [[package]] name = "rustls" version = "0.23.43" @@ -12241,6 +12346,28 @@ dependencies = [ "xattr", ] +[[package]] +name = "tar-codec" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ea42eb144d30fcbf32c26dfea8959175bb8335bfb7b18d20c6ed32edecf0551" +dependencies = [ + "archive-trait", + "tar-framing", + "thiserror 2.0.20", + "tokio", +] + +[[package]] +name = "tar-framing" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "783223a7a6590be4227cb821e7ce80372575511f67c824921aba0752d8ad5573" +dependencies = [ + "thiserror 2.0.20", + "tokio", +] + [[package]] name = "tcp-stream" version = "0.34.14" @@ -13520,7 +13647,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -13538,14 +13674,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -13563,48 +13716,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "1.0.4" @@ -13614,6 +13815,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags 2.13.1", + "windows-sys 0.52.0", +] + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/Cargo.toml b/Cargo.toml index 05ddc1110..11351dcea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -234,8 +234,11 @@ tokio-postgres-rustls = "0.14.0" # Utilities and Tools anyhow = "1.0.104" arc-swap = "1.9.2" -# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams. +# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork. astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" } +# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures. +tar-codec = "0.0.14" +tar-framing = "0.0.14" atoi = "3.1.0" atomic_enum = "0.3.0" aws-config = { version = "1.11.0" } diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 5558ffa44..312d2928d 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -49,5 +49,14 @@ rustfs-rio.workspace = true tokio = { workspace = true, features = ["io-util", "macros", "rt"] } thiserror = { workspace = true } +[dev-dependencies] +astral-tokio-tar = { workspace = true } +futures = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +sha2 = { workspace = true } +tar-codec = { workspace = true } +tar-framing = { workspace = true } + [lints] workspace = true diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md new file mode 100644 index 000000000..d9b558d81 --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md @@ -0,0 +1,24 @@ +# minio-go Snowball fixtures + +These request bodies are generated by +`github.com/minio/minio-go/v7.Client.PutObjectsSnowball` at the version pinned +in `generate/go.mod`. They cover the raw TAR and S2-compressed forms accepted by +RustFS Snowball extraction. + +The decoded TAR intentionally ends immediately after the final padded member +body because minio-go flushes, rather than closes, its TAR writer. The +compatibility test permits that shape only when the authenticated request body +is complete at the exact member boundary; it does not make incomplete TAR +terminators generally valid. + +Regenerate them from this directory with Go 1.25: + +```console +cd generate +go mod download +go run . -out .. +``` + +`manifest.json` records the input objects and SHA-256 digest of each captured +request body. Review changes to the manifest and binary fixtures together when +updating minio-go. diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod new file mode 100644 index 000000000..e2fd8f481 --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod @@ -0,0 +1,26 @@ +module rustfs.local/snowball-fixture + +go 1.25.0 + +require github.com/minio/minio-go/v7 v7.3.0 + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect +) diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum new file mode 100644 index 000000000..0027f37cd --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum @@ -0,0 +1,59 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU= +github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go new file mode 100644 index 000000000..b03052943 --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go @@ -0,0 +1,193 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const minioGoVersion = "v7.3.0" + +type fixtureManifest struct { + Generator string `json:"generator"` + MinioGo string `json:"minio_go"` + GeneratedAt string `json:"generated_at"` + Objects []fixtureObject `json:"objects"` + Archives []fixtureArchive `json:"archives"` +} + +type fixtureObject struct { + Key string `json:"key"` + Body string `json:"body"` + ModTime string `json:"mod_time"` + VersionID string `json:"version_id,omitempty"` + Headers map[string][]string `json:"headers,omitempty"` +} + +type fixtureArchive struct { + File string `json:"file"` + Compressed bool `json:"compressed"` + Length int `json:"length"` + SHA256 string `json:"sha256"` +} + +func objects() []fixtureObject { + return []fixtureObject{ + { + Key: "alpha.txt", + Body: "alpha-body", + ModTime: "2024-01-02T03:04:05Z", + VersionID: "018cc251-f400-7c22-9e8d-8b1800000001", + Headers: map[string][]string{ + "Content-Type": {"text/plain"}, + "X-Amz-Meta-Owner": {"snowball-fixture"}, + "X-Amz-Tagging": {"project=rustfs&source=minio-go"}, + }, + }, + { + Key: "nested/世界.txt", + Body: "bravo-body", + ModTime: "2024-01-02T03:05:05Z", + Headers: map[string][]string{ + "Content-Language": {"zh-CN"}, + "X-Amz-Meta-Note": {"unicode-path"}, + }, + }, + } +} + +func captureSnowball(compressed bool, specs []fixtureObject) ([]byte, error) { + body := make(chan []byte, 1) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + payload, err := io.ReadAll(request.Body) + if err != nil { + http.Error(writer, err.Error(), http.StatusInternalServerError) + return + } + body <- payload + writer.Header().Set("ETag", `"snowball-fixture"`) + writer.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{ + // The S3 authentication layer removes AWS streaming-signature framing + // before Snowball extraction sees the request body. Anonymous signing + // captures those decoded archive bytes directly. + Creds: credentials.NewStatic("", "", "", credentials.SignatureAnonymous), + Secure: false, + Region: "us-east-1", + }) + if err != nil { + return nil, fmt.Errorf("construct minio client: %w", err) + } + + input := make(chan minio.SnowballObject, len(specs)) + for _, spec := range specs { + modTime, err := time.Parse(time.RFC3339, spec.ModTime) + if err != nil { + return nil, fmt.Errorf("parse mod time for %q: %w", spec.Key, err) + } + headers := make(http.Header, len(spec.Headers)) + for name, values := range spec.Headers { + headers[name] = append([]string(nil), values...) + } + input <- minio.SnowballObject{ + Key: spec.Key, + Size: int64(len(spec.Body)), + ModTime: modTime, + Content: bytes.NewReader([]byte(spec.Body)), + VersionID: spec.VersionID, + Headers: headers, + } + } + close(input) + + err = client.PutObjectsSnowball(context.Background(), "fixture-bucket", minio.SnowballOptions{ + Opts: minio.PutObjectOptions{ + ContentType: "application/octet-stream", + }, + InMemory: true, + Compress: compressed, + }, input) + if err != nil { + return nil, fmt.Errorf("generate snowball request: %w", err) + } + return <-body, nil +} + +func main() { + outDir := flag.String("out", "..", "fixture output directory") + flag.Parse() + + specs := objects() + archives := make([]fixtureArchive, 0, 2) + for _, fixture := range []struct { + name string + compressed bool + }{ + {name: "snowball.tar"}, + {name: "snowball.tar.s2", compressed: true}, + } { + payload, err := captureSnowball(fixture.compressed, specs) + if err != nil { + panic(err) + } + path := filepath.Join(*outDir, fixture.name) + if err := os.WriteFile(path, payload, 0o644); err != nil { + panic(fmt.Errorf("write %s: %w", path, err)) + } + digest := sha256.Sum256(payload) + archives = append(archives, fixtureArchive{ + File: fixture.name, + Compressed: fixture.compressed, + Length: len(payload), + SHA256: hex.EncodeToString(digest[:]), + }) + } + + manifest := fixtureManifest{ + Generator: "github.com/minio/minio-go/v7.Client.PutObjectsSnowball", + MinioGo: minioGoVersion, + GeneratedAt: "2026-09-05T00:00:00Z", + Objects: specs, + Archives: archives, + } + payload, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + panic(err) + } + payload = append(payload, '\n') + path := filepath.Join(*outDir, "manifest.json") + if err := os.WriteFile(path, payload, 0o644); err != nil { + panic(fmt.Errorf("write %s: %w", path, err)) + } +} diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json new file mode 100644 index 000000000..85191f09c --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json @@ -0,0 +1,51 @@ +{ + "generator": "github.com/minio/minio-go/v7.Client.PutObjectsSnowball", + "minio_go": "v7.3.0", + "generated_at": "2026-09-05T00:00:00Z", + "objects": [ + { + "key": "alpha.txt", + "body": "alpha-body", + "mod_time": "2024-01-02T03:04:05Z", + "version_id": "018cc251-f400-7c22-9e8d-8b1800000001", + "headers": { + "Content-Type": [ + "text/plain" + ], + "X-Amz-Meta-Owner": [ + "snowball-fixture" + ], + "X-Amz-Tagging": [ + "project=rustfs\u0026source=minio-go" + ] + } + }, + { + "key": "nested/世界.txt", + "body": "bravo-body", + "mod_time": "2024-01-02T03:05:05Z", + "headers": { + "Content-Language": [ + "zh-CN" + ], + "X-Amz-Meta-Note": [ + "unicode-path" + ] + } + } + ], + "archives": [ + { + "file": "snowball.tar", + "compressed": false, + "length": 4096, + "sha256": "f00f2789dcb65b567f722f49cfdac9705e7bdac6c0badae75194327c32193d2e" + }, + { + "file": "snowball.tar.s2", + "compressed": true, + "length": 528, + "sha256": "f8a9d9aa9b9ccdfae24ded1bff3741aacb935f1457a252efc9266674ff13c992" + } + ] +} diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar new file mode 100644 index 0000000000000000000000000000000000000000..ea427f547ce3c6a1a9a3b518b2056bf14b60bc9a GIT binary patch literal 4096 zcmeHJJ#K?A6lUfWOx;XuV;~(oR9RZ7O*3TZ>VQM2h>gG`BxmT_o}~w>)N2$PXcDSy zP^A%G9>#vp@9*>H3CnLe^Ldh*aKNH?!AzAat@}>VK*jR=ll;)Bv<}=02kravx1*V&rQ^j*Nh$9vX-Tt-n3EV8V;Gs|i zk88mbnhLp|u_yxbFjrZ^b;oL2-|0+Q5QHM2aU!3&r)W}CVV+*4GD|#8za0XB)NQ9y zUlsisN^$2?h=!hrZ3N~7Lg0EBgKIwY!ElC#RevG<(u>w&d>ujl$iox3-(wzO4H26V z$8`uP?gE6M3pL|UFxeLVn?W9B;Cv_F^#?QQZ_d}p-m~s9Ab6^{KbRao1KNH8G@;R$ zL*(H7U9jVS3*(;uyQ})%GC@s-uKE9-i6CPEr`rV>jdkKSHy+UcSSn645qi?+AZBWz zW5*~Zt@`WZ{q6JhrEIL8ocGB;gRiPSZA1PjvIgF3tk)72@BJVCw+Slw-zr4U|3_Ku RW-?3IO7{HUO>+N%)h|U}$ol{Q literal 0 HcmV?d00001 diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2 b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2 new file mode 100644 index 0000000000000000000000000000000000000000..b9337a76637692871095feba8ac926b594131a31 GIT binary patch literal 528 zcmW;JL2DC17zW@kA<=b7p^I5CBC-cj6egLO%x>agYhw@D_>Y;RYcd}VYW|Q4X zljh_P=tZc-iy(sH#k)VjgI5Lb1wl{{5l>RLy}utX&-Wgk0@$EwZ?*>cJiNJ&i%+f1 zbw0l%cp#Frifep09`LG~D==~SGDnwM^d+MUW2_&)EX^NC(EL> znG}he%6R7S;SlwsxtbiJu5xxk*;sPcNtOH1UO749h5tb=z6G zvA>f^qFCMyTuhq2Pi=zw3}e*rDMi;sGeAv`G&`*V(Mj8cEu(0{`^j$;Zqovw7n8#k z4YqOS1j1~_oQA`gow*5aV6Z0pCOmisCN!5m8J`bqI6hBq!7}pVVAtrHz>Go#-oiO4 zQYC^KJ+1xBYbKN?XCEJbk~q{Xod->;Q(6HZfc)_Pe7OkP>OC&QDG!Ca7@&4nXN{Gs z@b7578!O>XW#q?!Kx3{3I(4j^25x7&eE8wu+w;BFqhBv8V4V!GfnFV3Rx``!>xqSC WT{a^*(CUm-z=LHk;X82!{=$E|_>r3c literal 0 HcmV?d00001 diff --git a/crates/zip/tests/snowball_tar_codec_compat.rs b/crates/zip/tests/snowball_tar_codec_compat.rs new file mode 100644 index 000000000..53e37afa8 --- /dev/null +++ b/crates/zip/tests/snowball_tar_codec_compat.rs @@ -0,0 +1,548 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::io::Cursor; + +use futures::StreamExt; +use rustfs_zip::CompressionFormat; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use tar_codec::{Archive as _, DecodePolicy, Member, MemberPayload as _, PaxDecodePolicy, PaxVendorExtensionPolicy, TarArchive}; +use tar_framing::{ + FrameError, FrameErrorInner, PaxKeyword, PaxRecord, PaxValue, StreamPolicy, UstarKind, + logical::{MemberExtensions, PaxState, TarReader}, +}; +use tokio::io::AsyncReadExt; + +const FIXTURE_ROOT: &str = "fixtures/snowball/minio-go-v7.3.0"; +const RAW_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar"); +const S2_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2"); +const MANIFEST: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/manifest.json"); + +#[derive(Debug, Deserialize)] +struct FixtureManifest { + generator: String, + minio_go: String, + generated_at: String, + objects: Vec, + archives: Vec, +} + +#[derive(Debug, Deserialize)] +struct FixtureObject { + key: String, + body: String, + mod_time: String, + #[serde(default)] + version_id: String, + #[serde(default)] + headers: BTreeMap>, +} + +#[derive(Debug, Deserialize)] +struct FixtureArchive { + file: String, + compressed: bool, + length: usize, + sha256: String, +} + +#[derive(Debug, Eq, PartialEq)] +struct ParsedMember { + path: String, + size: u64, + mtime: Option, + body: Vec, + minio_pax: BTreeMap>>, +} + +fn sha256_hex(bytes: &[u8]) -> String { + let mut encoded = String::with_capacity(64); + for byte in Sha256::digest(bytes) { + write!(&mut encoded, "{byte:02x}").expect("writing to a String should not fail"); + } + encoded +} + +async fn decode_s2(bytes: &[u8]) -> Vec { + let mut decoder = CompressionFormat::S2 + .get_decoder(Cursor::new(bytes.to_vec())) + .expect("S2 fixture decoder should be available"); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).await.expect("S2 fixture should decode"); + decoded +} + +async fn parse_with_tokio_tar(bytes: &[u8]) -> Vec { + let mut archive = tokio_tar::Archive::new(Cursor::new(bytes.to_vec())); + let mut entries = archive.entries().expect("tokio-tar should create an entry stream"); + let mut parsed = Vec::new(); + + while let Some(entry) = entries.next().await { + let mut entry = entry.expect("tokio-tar should parse the fixture member"); + let kind = entry.header().entry_type(); + if kind == tokio_tar::EntryType::XGlobalHeader { + continue; + } + + let path_bytes = entry.path_bytes().expect("tokio-tar should resolve the fixture path"); + let path = std::str::from_utf8(path_bytes.as_ref()) + .expect("fixture paths should be UTF-8") + .to_owned(); + let size = entry.effective_size(); + let mtime = entry.header().mtime().ok(); + let mut minio_pax = BTreeMap::new(); + if let Some(extensions) = entry + .pax_extensions() + .await + .expect("tokio-tar should parse local PAX records") + { + for extension in extensions { + let extension = extension.expect("fixture PAX record should be valid"); + let key = extension.key().expect("fixture PAX keys should be UTF-8"); + if key.starts_with("minio.") { + minio_pax.insert(key.to_owned(), Some(extension.value_bytes().to_vec())); + } + } + } + let mut body = Vec::new(); + entry + .read_to_end(&mut body) + .await + .expect("tokio-tar should read the fixture body"); + parsed.push(ParsedMember { + path, + size, + mtime, + body, + minio_pax, + }); + } + parsed +} + +fn effective_minio_pax(state: &PaxState<'_>, known_keywords: &mut Vec) -> BTreeMap>> { + for extension in state.extensions() { + for record in extension.records() { + let keyword = record.keyword(); + if matches!(&keyword, PaxKeyword::Vendor { vendor, .. } if vendor.as_ref() == "minio") + && !known_keywords.contains(&keyword) + { + known_keywords.push(keyword); + } + } + } + + known_keywords + .iter() + .filter_map(|keyword| { + let record = state.effective_record(keyword)?; + let PaxRecord::Vendor { vendor, name, value } = record else { + return None; + }; + let key = format!("{vendor}.{name}"); + let value = match value { + PaxValue::Value(value) => Some(value.to_vec()), + PaxValue::Deleted => None, + }; + Some((key, value)) + }) + .collect() +} + +fn effective_mtime(header_mtime: Option, extensions: &MemberExtensions<'_>) -> Option { + let MemberExtensions::Pax(state) = extensions else { + return header_mtime; + }; + match state.effective_record(&PaxKeyword::Mtime) { + Some(PaxRecord::Mtime(PaxValue::Value(value))) => Some(*value), + Some(PaxRecord::Mtime(PaxValue::Deleted)) => None, + _ => header_mtime, + } +} + +fn padded_member_end(position: u64, size: u64) -> u64 { + let padded_size = size.checked_add(511).expect("fixture member size should not overflow") / 512 * 512; + position + .checked_add(512) + .and_then(|position| position.checked_add(padded_size)) + .expect("fixture member end should not overflow") +} + +fn is_authenticated_footerless_end(error: &FrameError, last_member_end: Option, request_body_complete: bool) -> bool { + // The production gate must source `request_body_complete` from RustFS's + // length, checksum, and trailing-header validation state. + request_body_complete && matches!(&error.inner, FrameErrorInner::MissingEndMarker) && last_member_end == Some(error.position) +} + +fn candidate_snowball_decode_policy() -> DecodePolicy { + DecodePolicy::default() + .allow_gnu(true) + .allow_all_nul_numeric_fields(true) + .max_gnu_extension_size(1_048_576) + .pax_policy( + PaxDecodePolicy::default() + .max_extension_size(1_048_576) + .max_global_extensions_size(67_108_864) + .allow_global_pax_extensions(false) + .allow_non_utf8_pax_vendor_values(false) + .allow_duplicate_pax_records(false) + .allow_global_pax_member_metadata(false) + .vendor_extension_policy(PaxVendorExtensionPolicy::ignore(["minio"])), + ) +} + +async fn parse_with_tar_framing(bytes: &[u8]) -> (Vec, Option, Option) { + let policy = StreamPolicy::default() + .max_pax_extension_size(1024 * 1024) + .max_global_pax_extensions_size(4 * 1024 * 1024) + .max_gnu_extension_size(128 * 1024); + let mut reader = TarReader::new(Cursor::new(bytes.to_vec())).with_policy(policy); + let mut parsed = Vec::new(); + let mut known_minio_keywords = Vec::new(); + let mut last_member_end = None; + + loop { + let mut frame = match reader.next_frame().await { + Ok(Some(frame)) => frame, + Ok(None) => return (parsed, None, last_member_end), + Err(error) => return (parsed, Some(error), last_member_end), + }; + assert_eq!(frame.header.kind, UstarKind::Regular); + let path = String::from_utf8( + frame + .effective_path() + .expect("tar-framing should resolve the fixture path") + .into_owned(), + ) + .expect("fixture paths should be UTF-8"); + let size = frame.header.effective_size; + let mtime = effective_mtime(frame.header.mtime, &frame.extensions); + let minio_pax = match &frame.extensions { + MemberExtensions::Pax(state) => effective_minio_pax(state, &mut known_minio_keywords), + MemberExtensions::Gnu { .. } => BTreeMap::new(), + }; + let mut body = Vec::new(); + let mut chunk = Vec::new(); + while frame + .payload + .next_chunk(&mut chunk, 64 * 1024) + .await + .expect("tar-framing should read the fixture body") + { + body.extend_from_slice(&chunk); + } + last_member_end = Some(padded_member_end(frame.header.position, size)); + parsed.push(ParsedMember { + path, + size, + mtime, + body, + minio_pax, + }); + } +} + +#[test] +fn checked_in_fixtures_match_the_minio_go_manifest() { + let manifest: FixtureManifest = serde_json::from_slice(MANIFEST).expect("fixture manifest should be valid JSON"); + assert_eq!(manifest.generator, "github.com/minio/minio-go/v7.Client.PutObjectsSnowball"); + assert_eq!(manifest.minio_go, "v7.3.0"); + assert_eq!(manifest.generated_at, "2026-09-05T00:00:00Z"); + assert_eq!(manifest.objects.len(), 2); + assert_eq!(manifest.objects[0].key, "alpha.txt"); + assert_eq!(manifest.objects[0].body, "alpha-body"); + assert_eq!(manifest.objects[0].mod_time, "2024-01-02T03:04:05Z"); + assert_eq!(manifest.objects[0].version_id, "018cc251-f400-7c22-9e8d-8b1800000001"); + assert_eq!( + manifest.objects[0].headers.get("X-Amz-Meta-Owner"), + Some(&vec!["snowball-fixture".to_owned()]) + ); + + for archive in &manifest.archives { + let bytes = match archive.file.as_str() { + "snowball.tar" => RAW_FIXTURE, + "snowball.tar.s2" => S2_FIXTURE, + file => panic!("unexpected archive in {FIXTURE_ROOT}/manifest.json: {file}"), + }; + assert_eq!(bytes.len(), archive.length); + assert_eq!(sha256_hex(bytes), archive.sha256); + assert_eq!(archive.compressed, archive.file.ends_with(".s2")); + } +} + +#[tokio::test] +async fn minio_go_raw_and_s2_fixtures_have_identical_footerless_tar_data() { + assert_eq!(decode_s2(S2_FIXTURE).await, RAW_FIXTURE); + assert_eq!(RAW_FIXTURE.len() % 512, 0); + assert!(RAW_FIXTURE.len() >= 1024); + assert!( + !RAW_FIXTURE[RAW_FIXTURE.len() - 1024..].iter().all(|byte| *byte == 0), + "minio-go Flush output should not contain the standard two-block terminator" + ); +} + +#[tokio::test] +async fn tar_framing_matches_tokio_tar_before_rejecting_the_missing_terminator() { + let expected = parse_with_tokio_tar(RAW_FIXTURE).await; + let (actual, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await; + let error = error.expect("footerless minio-go fixture should fail strict termination"); + + assert_eq!(actual, expected); + assert_eq!( + actual, + [ + ParsedMember { + path: "alpha.txt".to_owned(), + size: 10, + mtime: Some(1_704_164_645), + body: b"alpha-body".to_vec(), + minio_pax: BTreeMap::from([ + ("minio.metadata.Content-Type".to_owned(), Some(b"text/plain".to_vec()),), + ("minio.metadata.X-Amz-Meta-Owner".to_owned(), Some(b"snowball-fixture".to_vec()),), + ( + "minio.metadata.X-Amz-Tagging".to_owned(), + Some(b"project=rustfs&source=minio-go".to_vec()), + ), + ("minio.versionId".to_owned(), Some(b"018cc251-f400-7c22-9e8d-8b1800000001".to_vec()),), + ]), + }, + ParsedMember { + path: "nested/世界.txt".to_owned(), + size: 10, + mtime: Some(1_704_164_705), + body: b"bravo-body".to_vec(), + minio_pax: BTreeMap::from([ + ("minio.metadata.Content-Language".to_owned(), Some(b"zh-CN".to_vec()),), + ("minio.metadata.X-Amz-Meta-Note".to_owned(), Some(b"unicode-path".to_vec()),), + ]), + }, + ] + ); + assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker)); + assert_eq!( + error.position, + u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64") + ); + assert_eq!(last_member_end, Some(error.position)); +} + +#[tokio::test] +async fn footerless_compatibility_requires_authenticated_eof_at_the_member_boundary() { + let (_, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await; + let error = error.expect("the real fixture should be footerless"); + assert!(is_authenticated_footerless_end(&error, last_member_end, true)); + assert!(!is_authenticated_footerless_end(&error, last_member_end, false)); + + let mut one_zero_block = RAW_FIXTURE.to_vec(); + one_zero_block.extend([0; 512]); + let (_, error, last_member_end) = parse_with_tar_framing(&one_zero_block).await; + let error = error.expect("one zero block is not a valid TAR terminator"); + assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker)); + assert_eq!( + last_member_end, + Some(u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64")) + ); + assert_eq!( + error.position, + u64::try_from(one_zero_block.len()).expect("fixture length should fit in u64") + ); + assert!(!is_authenticated_footerless_end(&error, last_member_end, true)); +} + +#[tokio::test] +async fn tar_codec_policy_accepts_only_the_explicit_minio_vendor_namespace() { + let default_error = match TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec())).members().next().await { + Err(error) => error, + Ok(_) => panic!("the default policy should reject minio vendor records"), + }; + assert!(default_error.to_string().contains("pax vendor extension minio.")); + + let mut members = TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec())) + .with_policy(candidate_snowball_decode_policy()) + .members(); + let mut bodies = Vec::new(); + loop { + let member = match members.next().await { + Ok(Some(member)) => member, + Ok(None) => panic!("footerless minio-go fixture should not report a valid archive end"), + Err(error) => { + assert!(error.to_string().contains("missing two-block end-of-archive marker")); + break; + } + }; + let Member::File { mut payload, .. } = member else { + panic!("fixture should contain only regular files"); + }; + let mut body = Vec::new(); + let mut chunk = Vec::new(); + while payload + .next_chunk(&mut chunk, 64 * 1024) + .await + .expect("tar-codec should read the fixture body") + { + body.extend_from_slice(&chunk); + } + bodies.push(body); + } + assert_eq!(bodies, [b"alpha-body".to_vec(), b"bravo-body".to_vec()]); + assert!( + members + .next() + .await + .expect("the member cursor should be fused after an error") + .is_none() + ); +} + +fn pax_record(key: &str, value: &str) -> Vec { + let payload = format!("{key}={value}\n"); + let mut len = payload.len() + 3; + loop { + let record = format!("{len} {payload}"); + if record.len() == len { + return record.into_bytes(); + } + len = record.len(); + } +} + +async fn append_pax_header( + builder: &mut tokio_tar::Builder>>, + entry_type: tokio_tar::EntryType, + records: &[(&str, &str)], +) { + let mut payload = Vec::new(); + for (key, value) in records { + payload.extend(pax_record(key, value)); + } + let mut header = tokio_tar::Header::new_ustar(); + header.set_entry_type(entry_type); + header.set_size(u64::try_from(payload.len()).expect("PAX test payload should fit in u64")); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload)) + .await + .expect("PAX test header should be written"); +} + +async fn append_regular(builder: &mut tokio_tar::Builder>>, path: &str) { + let body = path.as_bytes(); + let mut header = tokio_tar::Header::new_ustar(); + header.set_entry_type(tokio_tar::EntryType::Regular); + header.set_size(u64::try_from(body.len()).expect("test member body should fit in u64")); + header.set_mode(0o644); + header.set_mtime(1_704_164_645); + header.set_cksum(); + builder + .append_data(&mut header, path, Cursor::new(body)) + .await + .expect("ordinary test member should be written"); +} + +async fn archive_with_local_pax(records: &[(&str, &str)]) -> Vec { + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, records).await; + append_regular(&mut builder, "member.txt").await; + builder.into_inner().await.expect("policy archive should finish").into_inner() +} + +#[tokio::test] +async fn candidate_policy_rejects_unknown_vendor_and_duplicate_pax_records() { + let unknown_vendor = archive_with_local_pax(&[("acme.metadata.owner", "mallory")]).await; + let error = match TarArchive::new(Cursor::new(unknown_vendor)) + .with_policy(candidate_snowball_decode_policy()) + .members() + .next() + .await + { + Err(error) => error, + Ok(_) => panic!("the candidate Snowball policy should reject unknown vendors"), + }; + assert!( + error + .to_string() + .contains("pax vendor extension acme.metadata.owner is not allowed") + ); + + let duplicate = archive_with_local_pax(&[ + ("minio.metadata.x-amz-meta-owner", "first"), + ("minio.metadata.x-amz-meta-owner", "second"), + ]) + .await; + let error = match TarArchive::new(Cursor::new(duplicate)) + .with_policy(candidate_snowball_decode_policy()) + .members() + .next() + .await + { + Err(error) => error, + Ok(_) => panic!("the candidate Snowball policy should reject duplicate PAX records"), + }; + assert!( + error + .to_string() + .contains("pax extended header contains duplicate record minio.metadata.x-amz-meta-owner") + ); +} + +#[tokio::test] +async fn global_minio_pax_inheritance_is_an_explicit_migration_difference() { + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + append_pax_header( + &mut builder, + tokio_tar::EntryType::XGlobalHeader, + &[("minio.metadata.x-amz-meta-owner", "global")], + ) + .await; + append_pax_header( + &mut builder, + tokio_tar::EntryType::XHeader, + &[("minio.metadata.x-amz-meta-owner", "local")], + ) + .await; + append_regular(&mut builder, "local.txt").await; + append_regular(&mut builder, "inherited.txt").await; + let archive = builder + .into_inner() + .await + .expect("precedence archive should finish") + .into_inner(); + + let legacy = parse_with_tokio_tar(&archive).await; + let (framing, error, _) = parse_with_tar_framing(&archive).await; + assert!(error.is_none()); + assert_eq!(legacy.len(), 2); + assert_eq!(framing.len(), 2); + + let owner_key = "minio.metadata.x-amz-meta-owner"; + assert_eq!(legacy[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec()))); + assert!(!legacy[1].minio_pax.contains_key(owner_key)); + assert_eq!(framing[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec()))); + assert_eq!(framing[1].minio_pax.get(owner_key), Some(&Some(b"global".to_vec()))); + + let error = match TarArchive::new(Cursor::new(archive)) + .with_policy(candidate_snowball_decode_policy()) + .members() + .next() + .await + { + Err(error) => error, + Ok(_) => panic!("the candidate Snowball policy should reject global PAX state"), + }; + assert!(error.to_string().contains("global pax extended headers are not allowed")); +} diff --git a/deny.toml b/deny.toml index fc5d25916..cfecab82f 100644 --- a/deny.toml +++ b/deny.toml @@ -37,8 +37,8 @@ unknown-git = "deny" allow-registry = ["https://github.com/rust-lang/crates.io-index"] allow-git = [ # Temporary tokio-tar fork pinned to the reviewed parser limits, - # cancellation safety, and error-fusing change while - # astral-sh/tokio-tar#118 awaits an upstream release. + # cancellation safety, and error-fusing change while Snowball is + # prototyped against tar-codec and Swift retains its current reader. # owner: cxymds review: 2026-10 "https://github.com/cxymds/tokio-tar.git", # Official s3s repository. Temporarily pinned to the merged generic REST diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 6472481a0..924193b6e 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -13,7 +13,7 @@ - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation. - `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains. -- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release. +- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on precedence-resolved MinIO PAX metadata; per-entry and cumulative extension limits; a physical-entry limit; cancellation-safe parsing and ownership of large streamed members; fused streams after errors; and compatibility with minio-go streams that omit the two-block terminator. Swift bulk extraction also uses the same fork. Keep the reviewed pin while the Snowball path is prototyped against tar-codec/tar-framing. Remove it only after a released API exposes the effective allowed vendor records, RustFS provides a cancellation-safe handoff for borrowed member payloads, footerless input is accepted solely when authenticated request framing proves EOF immediately after a complete member, the existing resource-limit, cancellation, error-fuse, and real minio-go fixtures pass against the replacement, and Swift no longer depends on the fork. - `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources. - `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources. - `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive.