diff --git a/Cargo.lock b/Cargo.lock index 0fb09e78e..82f66641e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10489,15 +10489,10 @@ dependencies = [ name = "rustfs-zip" version = "1.0.0-rc.2" dependencies = [ - "astral-tokio-tar", "async-compression", - "criterion", "hotpath", - "tempfile", "thiserror 2.0.20", "tokio", - "tokio-stream", - "zip", ] [[package]] diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 54cdd9547..01ec5d84e 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -20,7 +20,7 @@ repository.workspace = true rust-version.workspace = true version.workspace = true homepage.workspace = true -description = "ZIP file handling for RustFS, providing support for reading and writing ZIP archives." +description = "Archive format detection and async stream decoders for RustFS." keywords = ["zip", "compression", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "compression"] documentation = "https://docs.rs/rustfs-zip/latest/rustfs_zip/" @@ -28,10 +28,6 @@ documentation = "https://docs.rs/rustfs-zip/latest/rustfs_zip/" [lib] doctest = false -[[bench]] -name = "zip_benchmark" -harness = false - [features] default = [] hotpath = ["hotpath/hotpath", "hotpath/tokio"] @@ -48,16 +44,8 @@ async-compression = { workspace = true, features = [ "zstd", "xz", ] } -tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread"] } -tokio-stream = { workspace = true } -astral-tokio-tar = { workspace = true } +tokio = { workspace = true, features = ["io-util", "macros", "rt"] } thiserror = { workspace = true } -zip = { workspace = true } - -[dev-dependencies] -criterion = { workspace = true, features = ["html_reports"] } -tempfile = { workspace = true } - [lints] workspace = true diff --git a/crates/zip/README.md b/crates/zip/README.md index 78e105f5f..a9b7865d5 100644 --- a/crates/zip/README.md +++ b/crates/zip/README.md @@ -1,9 +1,9 @@ [![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com) -# RustFS Zip - Archive And Compression Primitives +# RustFS Zip - Archive Format Detection And Stream Decoding

- High-performance compression and archiving for RustFS object storage + Archive format detection and async stream decoders for RustFS object storage

@@ -17,53 +17,23 @@ ## 📖 Overview -**RustFS Zip** provides archive and compression primitives for the [RustFS](https://rustfs.com) distributed object storage system. Today it is primarily used by RustFS archive extract flows to: +**RustFS Zip** provides the archive primitives used by the [RustFS](https://rustfs.com) archive extract flow: -- identify archive/compression formats by extension -- stream tar and tar+compression inputs through async decoders -- provide small ZIP read/write helpers for local archive workflows +- identify a compression format from an archive extension +- wrap an async reader in the matching stream decoder +- carry the shared default archive guardrails ## Current Features -- A clearer type model with: - - `CompressionCodec` for stream codecs - - `ArchiveKind` for container families - - `ArchiveFormat` for concrete archive/container combinations -- Async stream codecs for `gzip`, `bzip2`, `zlib`, `xz`, and `zstd` -- Tar archive iteration over async readers through `read_archive_entries()` / `extract_tar_entries()` -- Archive guardrails through `ArchiveLimits` for entry count, entry size, total unpacked size, and path length -- In-memory compression helpers for payload round-trip workflows -- Blocking ZIP create/extract helpers for local archive files -- ZIP helper metadata via `ZipEntry`, including: - - `compression_method` - - `archive_kind` - - `format` - - `unix_mode` -- ZIP helper options via `ZipWriteOptions`, including: - - `compression_level` - - `create_directory_entries` - -## Compatibility - -- `CompressionFormat` is retained as a compatibility layer for existing callers -- New code should prefer `ArchiveFormat`, `ArchiveKind`, and `CompressionCodec` when expressing archive semantics - -## ZIP Helper Scope - -The file-based ZIP helper APIs are best suited for: - -- local archive import/export flows -- admin-side packaging helpers -- test fixtures and tooling - -They are not intended to be a remote streaming ZIP access engine. +- `CompressionFormat::from_extension()` for extension-based format detection, including tar-family suffixes such as `tgz`, `tbz2`, `txz`, and `tzst` +- `CompressionFormat::get_decoder()` for async stream decoding of `gzip`, `bzip2`, `zlib`, `xz`, and `zstd`, plus a pass-through reader for plain `tar` +- `ArchiveLimits` with the default entry count, entry size, total unpacked size, and path length guardrails ## Current Boundaries -- ZIP is supported via file-based helper APIs, not the tar-family async stream APIs -- Tar-family stream APIs are intended for `tar`, `tar.gz`, `tar.bz2`, `tar.xz`, `tar.zst`, and similar compressed tar flows -- Default archive guardrails are intentionally conservative and do not replace higher-level RustFS object-path validation -- This crate does not currently implement a general-purpose parallel archive engine +- ZIP has no stream decoder: `get_decoder()` rejects `CompressionFormat::Zip`, because ZIP needs central-directory semantics that a forward-only stream cannot provide +- This crate detects formats and hands back decoders; archive iteration, entry writing, and extraction to disk belong to the caller +- `ArchiveLimits` carries the values only; enforcement and the resulting protocol error belong to the caller - Archive extraction safety policy remains the responsibility of the RustFS caller for object-store flows ## 📚 Documentation diff --git a/crates/zip/benches/zip_benchmark.rs b/crates/zip/benches/zip_benchmark.rs deleted file mode 100644 index c7047918b..000000000 --- a/crates/zip/benches/zip_benchmark.rs +++ /dev/null @@ -1,416 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use rustfs_zip::{ - ArchiveLimits, CompressionFormat, CompressionLevel, ZipWriteOptions, create_zip_with_options, extract_tar_entries, - extract_zip_to_path_with_limits, extract_zip_with_limits, -}; -use std::hint::black_box; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; -use tempfile::tempdir; -use tokio::runtime::Builder; -use tokio_tar::{Builder as TarBuilder, Header}; -use zip::ZipArchive; - -fn build_runtime() -> tokio::runtime::Runtime { - Builder::new_current_thread() - .enable_all() - .build() - .expect("build tokio runtime for rustfs-zip benchmarks") -} - -async fn build_tar_payload(entry_count: usize, payload_size: usize) -> Vec { - let sink = tokio::io::duplex(64 * 1024); - let (writer, mut reader) = sink; - let write_task = tokio::spawn(async move { - let mut builder = TarBuilder::new(writer); - let payload = vec![b'a'; payload_size]; - for index in 0..entry_count { - let mut header = Header::new_gnu(); - header.set_size(payload.len() as u64); - header.set_mode(0o644); - header.set_cksum(); - builder - .append_data(&mut header, format!("entry-{index}.txt"), &payload[..]) - .await - .expect("append tar benchmark entry"); - } - builder.finish().await.expect("finish tar benchmark archive"); - }); - - let mut output = Vec::new(); - tokio::io::copy(&mut reader, &mut output) - .await - .expect("read tar benchmark archive"); - write_task.await.expect("join tar writer task"); - output -} - -async fn build_compressed_tar_payload(format: CompressionFormat, entry_count: usize, payload_size: usize) -> Vec { - let tar_payload = build_tar_payload(entry_count, payload_size).await; - rustfs_zip::Compressor::new(format) - .compress(&tar_payload) - .await - .expect("compress tar benchmark payload") -} - -fn bench_tar_family_extract(c: &mut Criterion) { - let runtime = build_runtime(); - let mut group = c.benchmark_group("zip_tar_family_extract"); - - for (name, format, entry_count, payload_size) in [ - ("tar_gzip_small_many", CompressionFormat::Gzip, 64usize, 256usize), - ("tar_zstd_medium", CompressionFormat::Zstd, 16usize, 16 * 1024usize), - ] { - let payload = runtime.block_on(build_compressed_tar_payload(format, entry_count, payload_size)); - group.throughput(Throughput::Bytes(payload.len() as u64)); - group.bench_with_input(BenchmarkId::new(name, payload.len()), &payload, |b, payload| { - b.iter(|| { - runtime.block_on(async { - let seen = Arc::new(AtomicUsize::new(0)); - let seen_ref = Arc::clone(&seen); - extract_tar_entries(std::io::Cursor::new(payload.clone()), format, move |_entry| { - let seen_ref = Arc::clone(&seen_ref); - async move { - seen_ref.fetch_add(1, Ordering::Relaxed); - Ok(()) - } - }) - .await - .expect("extract tar benchmark payload"); - black_box(seen.load(Ordering::Relaxed)); - }); - }); - }); - } - - group.finish(); -} - -fn bench_zip_helper_round_trip(c: &mut Criterion) { - let runtime = build_runtime(); - let mut group = c.benchmark_group("zip_helper_round_trip"); - - let zip_matrix = [ - ("stored_flat_32x128", CompressionLevel::Fastest, 32usize, 128usize, "flat"), - ("stored_nested_32x256", CompressionLevel::Fastest, 32usize, 256usize, "nested"), - ("stored_flat_256x128", CompressionLevel::Fastest, 256usize, 128usize, "flat"), - ("deflated_flat_32x1k", CompressionLevel::Best, 32usize, 1024usize, "flat"), - ("deflated_nested_256x1k", CompressionLevel::Best, 256usize, 1024usize, "nested"), - ("deflated_deep_1024x4k", CompressionLevel::Best, 1024usize, 4 * 1024usize, "deep"), - ]; - - for (name, compression_level, file_count, payload_size, layout) in zip_matrix { - let files = (0..file_count) - .map(|index| { - let path = match layout { - "flat" => format!("file-{index}.txt"), - "nested" => format!("batch-{}/file-{index}.txt", index % 8), - "deep" => format!("lvl1/lvl2-{}/lvl3-{}/file-{index}.txt", index % 16, index % 32), - _ => format!("file-{index}.txt"), - }; - (path, vec![b'b'; payload_size]) - }) - .collect::>(); - let total_bytes = (file_count * payload_size) as u64; - group.throughput(Throughput::Bytes(total_bytes)); - - group.bench_with_input(BenchmarkId::new(name, total_bytes), &files, |b, files| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - runtime.block_on(async { - create_zip_with_options( - &zip_path, - files.clone(), - ZipWriteOptions { - compression_level, - create_directory_entries: true, - }, - ) - .await - .expect("create zip benchmark archive"); - - let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("extract zip benchmark archive"); - black_box(entries.len()); - }); - }); - }); - } - - group.finish(); -} - -fn bench_zip_helper_hotspot_breakdown(c: &mut Criterion) { - let runtime = build_runtime(); - let mut group = c.benchmark_group("zip_helper_hotspot_breakdown"); - let files = (0..32) - .map(|index| (format!("batch/file-{index}.txt"), vec![b'c'; 256])) - .collect::>(); - let total_bytes = (32 * 256) as u64; - group.throughput(Throughput::Bytes(total_bytes)); - - group.bench_function("fs_setup_cleanup_only", |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - black_box((zip_path, extract_path)); - }); - }); - - group.bench_function("zip_create_only_stored_small", |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - runtime.block_on(async { - create_zip_with_options( - &zip_path, - files.clone(), - ZipWriteOptions { - compression_level: CompressionLevel::Fastest, - create_directory_entries: true, - }, - ) - .await - .expect("create zip benchmark archive"); - }); - }); - }); - - let payload_for_extract = { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - runtime.block_on(async { - create_zip_with_options( - &zip_path, - files.clone(), - ZipWriteOptions { - compression_level: CompressionLevel::Fastest, - create_directory_entries: true, - }, - ) - .await - .expect("prepare zip benchmark extract payload"); - }); - std::fs::read(&zip_path).expect("read benchmark zip payload") - }; - - group.bench_function("zip_extract_only_stored_small", |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - std::fs::write(&zip_path, &payload_for_extract).expect("write benchmark zip payload"); - runtime.block_on(async { - let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("extract zip benchmark archive"); - black_box(entries.len()); - }); - }); - }); - - group.bench_function("zip_extract_only_stored_small_summary_only", |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - std::fs::write(&zip_path, &payload_for_extract).expect("write benchmark zip payload"); - runtime.block_on(async { - let summary = extract_zip_to_path_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("extract zip benchmark summary path"); - black_box(summary.entry_count); - }); - }); - }); - - group.bench_function("zip_reader_only_stored_small", |b| { - b.iter(|| { - let cursor = std::io::Cursor::new(payload_for_extract.clone()); - let mut archive = ZipArchive::new(cursor).expect("open zip archive for reader-only benchmark"); - let mut total_bytes = 0usize; - for index in 0..archive.len() { - let mut zip_file = archive.by_index(index).expect("access zip entry by index"); - let enclosed_name = zip_file - .enclosed_name() - .expect("resolve enclosed zip entry name") - .to_string_lossy() - .replace('\\', "/"); - let size = zip_file.size(); - assert!(!enclosed_name.is_empty(), "zip reader-only benchmark expects non-empty names"); - assert!( - size <= ArchiveLimits::default().max_entry_size, - "zip reader-only benchmark expects small entries" - ); - if !zip_file.is_dir() { - let mut sink = [0_u8; 256]; - let bytes_read = - std::io::Read::read(&mut zip_file, &mut sink).expect("read zip entry payload for reader-only benchmark"); - total_bytes += bytes_read; - } - } - black_box(total_bytes); - }); - }); - - group.bench_function("file_write_only_stored_small", |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let extract_path = temp.path().join("extract"); - std::fs::create_dir_all(&extract_path).expect("create extract dir for file-write-only benchmark"); - let mut total_bytes = 0usize; - for index in 0..32 { - let path = extract_path.join(format!("file-{index}.txt")); - std::fs::write(&path, [b'c'; 256]).expect("write small file for file-write-only benchmark"); - total_bytes += 256; - } - black_box(total_bytes); - }); - }); - - group.finish(); -} - -fn build_object_archive_files( - metadata_count: usize, - metadata_size: usize, - payload_count: usize, - payload_size: usize, -) -> Vec<(String, Vec)> { - let mut files = Vec::with_capacity(metadata_count * 2 + payload_count); - - for index in 0..metadata_count { - let key_prefix = format!( - "bucket-a/shard-{}/tenant-{}/dataset-{}/object-{index:04}", - index % 8, - index % 16, - index % 32 - ); - files.push(( - format!("{key_prefix}/meta.json"), - format!( - "{{\"key\":\"object-{index:04}\",\"etag\":\"{:032x}\",\"size\":{},\"content_type\":\"application/octet-stream\"}}", - index, - payload_size - ) - .into_bytes(), - )); - files.push((format!("{key_prefix}/tags.txt"), vec![b'm'; metadata_size])); - } - - for index in 0..payload_count { - let payload_prefix = format!( - "bucket-a/shard-{}/tenant-{}/dataset-{}/object-{index:04}", - index % 8, - index % 16, - index % 32 - ); - files.push((format!("{payload_prefix}/part-00000.bin"), vec![b'p'; payload_size])); - } - - files -} - -fn bench_zip_object_archive_extract(c: &mut Criterion) { - let runtime = build_runtime(); - let mut group = c.benchmark_group("zip_object_archive_extract"); - - for (name, compression_level, metadata_count, metadata_size, payload_count, payload_size) in [ - ( - "stored_metadata_heavy_384m_24p", - CompressionLevel::Fastest, - 384usize, - 192usize, - 24usize, - 32 * 1024usize, - ), - ( - "deflated_mixed_192m_32p", - CompressionLevel::Best, - 192usize, - 256usize, - 32usize, - 64 * 1024usize, - ), - ] { - let files = build_object_archive_files(metadata_count, metadata_size, payload_count, payload_size); - let total_bytes = files.iter().map(|(_, payload)| payload.len() as u64).sum::(); - let payload = { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("object-archive.zip"); - runtime.block_on(async { - create_zip_with_options( - &zip_path, - files.clone(), - ZipWriteOptions { - compression_level, - create_directory_entries: true, - }, - ) - .await - .expect("create object archive benchmark payload"); - }); - std::fs::read(&zip_path).expect("read object archive benchmark payload") - }; - - group.throughput(Throughput::Bytes(total_bytes)); - group.bench_function(BenchmarkId::new("extract_full", name), |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - std::fs::write(&zip_path, &payload).expect("write object archive benchmark payload"); - runtime.block_on(async { - let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("extract object archive benchmark payload"); - black_box(entries.len()); - }); - }); - }); - - group.bench_function(BenchmarkId::new("extract_summary_only", name), |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - std::fs::write(&zip_path, &payload).expect("write object archive benchmark payload"); - runtime.block_on(async { - let summary = extract_zip_to_path_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("extract object archive benchmark summary"); - black_box(summary.file_count); - }); - }); - }); - } - - group.finish(); -} - -criterion_group!( - benches, - bench_tar_family_extract, - bench_zip_helper_round_trip, - bench_zip_helper_hotspot_breakdown, - bench_zip_object_archive_extract -); -criterion_main!(benches); diff --git a/crates/zip/src/lib.rs b/crates/zip/src/lib.rs index d61968fbb..82aa8867a 100644 --- a/crates/zip/src/lib.rs +++ b/crates/zip/src/lib.rs @@ -13,21 +13,8 @@ // limitations under the License. use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZlibDecoder, ZstdDecoder}; -use async_compression::tokio::write::{BzEncoder, GzipEncoder, XzEncoder, ZlibEncoder, ZstdEncoder}; -use std::collections::HashSet; -use std::future::Future; -use std::io::{Read, Write}; -use std::path::{Component, Path, PathBuf}; -use std::pin::Pin; -use std::sync::{Arc, Mutex}; -use std::task::{Context, Poll}; use thiserror::Error; -use tokio::fs::File; -use tokio::io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, BufWriter}; -use tokio::task::spawn_blocking; -use tokio_stream::StreamExt; -use tokio_tar::Archive; -use zip::{CompressionMethod, ZipArchive, ZipWriter, write::SimpleFileOptions}; +use tokio::io::{AsyncRead, BufReader}; pub type Result = std::result::Result; @@ -38,51 +25,6 @@ pub enum ZipError { format: CompressionFormat, operation: &'static str, }, - #[error("invalid compression level {0}: value exceeds i32::MAX")] - InvalidCompressionLevel(u32), - #[error("unsafe archive entry path: {0}")] - UnsafeEntryPath(String), - #[error("archive entry path length {length} exceeds limit {limit}: {path}")] - EntryPathTooLong { path: String, length: usize, limit: usize }, - #[error("archive entry count {count} exceeds limit {limit}")] - EntryCountLimitExceeded { count: usize, limit: usize }, - #[error("archive entry '{path}' size {size} exceeds limit {limit}")] - EntrySizeLimitExceeded { path: String, size: u64, limit: u64 }, - #[error("archive total unpacked size {size} exceeds limit {limit}")] - TotalUnpackedSizeLimitExceeded { size: u64, limit: u64 }, - #[error(transparent)] - Io(#[from] io::Error), - #[error(transparent)] - Zip(#[from] zip::result::ZipError), - #[error(transparent)] - Join(#[from] tokio::task::JoinError), -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum CompressionCodec { - Gzip, - Bzip2, - Xz, - Zlib, - Zstd, -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum ArchiveKind { - Tar, - Zip, -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum ArchiveFormat { - Tar, - TarGzip, - TarBzip2, - TarXz, - TarZlib, - TarZstd, - Zip, - Unknown, } #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -97,35 +39,9 @@ pub enum CompressionFormat { Unknown, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum CompressionLevel { - Fastest, - Best, - #[default] - Default, - Level(u32), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ZipEntry { - pub name: String, - pub size: u64, - pub compressed_size: u64, - pub is_dir: bool, - pub compression_method: String, - pub archive_kind: ArchiveKind, - pub format: ArchiveFormat, - pub unix_mode: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct ZipExtractSummary { - pub entry_count: usize, - pub directory_count: usize, - pub file_count: usize, - pub total_unpacked_size: u64, -} - +/// Archive guardrails. The values are carried here so every archive caller +/// shares one default policy; enforcement belongs to the caller, which maps a +/// breach onto its own protocol error. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ArchiveLimits { pub max_entries: usize, @@ -147,103 +63,23 @@ impl Default for ArchiveLimits { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ZipWriteOptions { - pub compression_level: CompressionLevel, - pub create_directory_entries: bool, -} - -impl Default for ZipWriteOptions { - fn default() -> Self { - Self { - compression_level: CompressionLevel::Default, - create_directory_entries: false, - } - } -} - -const SMALL_ZIP_EXTRACT_FAST_PATH_LIMIT: u64 = 8 * 1024; - -#[derive(Clone, Default)] -struct SharedBuffer { - inner: Arc>>, -} - -impl SharedBuffer { - fn into_vec(self) -> Vec { - self.inner.lock().expect("shared in-memory writer lock poisoned").clone() - } -} - -impl AsyncWrite for SharedBuffer { - fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll> { - let mut inner = self - .inner - .lock() - .map_err(|_| io::Error::other("shared in-memory writer lock poisoned"))?; - inner.extend_from_slice(buf); - Poll::Ready(Ok(buf.len())) - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } -} - impl CompressionFormat { + /// Map an archive extension onto the stream codec needed to read it. + /// Tar-family suffixes (`tgz`, `tbz2`, `txz`, `tzst`, ...) resolve to their + /// codec because the tar container itself is read from the decoded stream. pub fn from_extension(ext: &str) -> Self { - Self::from_archive_format(ArchiveFormat::from_extension(ext)) - } - - pub fn from_archive_format(format: ArchiveFormat) -> Self { - match format { - ArchiveFormat::TarGzip => CompressionFormat::Gzip, - ArchiveFormat::TarBzip2 => CompressionFormat::Bzip2, - ArchiveFormat::TarXz => CompressionFormat::Xz, - ArchiveFormat::TarZlib => CompressionFormat::Zlib, - ArchiveFormat::TarZstd => CompressionFormat::Zstd, - ArchiveFormat::Tar => CompressionFormat::Tar, - ArchiveFormat::Zip => CompressionFormat::Zip, - ArchiveFormat::Unknown => CompressionFormat::Unknown, + match ext.to_ascii_lowercase().as_str() { + "gz" | "gzip" | "tgz" => CompressionFormat::Gzip, + "bz2" | "bzip2" | "tbz" | "tbz2" => CompressionFormat::Bzip2, + "xz" | "txz" => CompressionFormat::Xz, + "zlib" | "zz" => CompressionFormat::Zlib, + "zst" | "zstd" | "tzst" => CompressionFormat::Zstd, + "tar" => CompressionFormat::Tar, + "zip" => CompressionFormat::Zip, + _ => CompressionFormat::Unknown, } } - pub fn archive_format_from_path>(path: P) -> ArchiveFormat { - ArchiveFormat::from_path(path) - } - - pub fn archive_kind(&self) -> Option { - match self { - CompressionFormat::Tar => Some(ArchiveKind::Tar), - CompressionFormat::Zip => Some(ArchiveKind::Zip), - CompressionFormat::Gzip - | CompressionFormat::Bzip2 - | CompressionFormat::Xz - | CompressionFormat::Zlib - | CompressionFormat::Zstd - | CompressionFormat::Unknown => None, - } - } - - pub fn compression_codec(&self) -> Option { - match self { - CompressionFormat::Gzip => Some(CompressionCodec::Gzip), - CompressionFormat::Bzip2 => Some(CompressionCodec::Bzip2), - CompressionFormat::Xz => Some(CompressionCodec::Xz), - CompressionFormat::Zlib => Some(CompressionCodec::Zlib), - CompressionFormat::Zstd => Some(CompressionCodec::Zstd), - CompressionFormat::Tar | CompressionFormat::Zip | CompressionFormat::Unknown => None, - } - } - - pub fn from_path>(path: P) -> Self { - Self::from_archive_format(ArchiveFormat::from_path(path)) - } - pub fn extension(&self) -> &'static str { match self { CompressionFormat::Gzip => "gz", @@ -257,10 +93,6 @@ impl CompressionFormat { } } - pub fn is_supported(&self) -> bool { - !matches!(self, CompressionFormat::Unknown) - } - pub fn get_decoder(&self, input: R) -> Result> where R: AsyncRead + Send + Unpin + 'static, @@ -290,625 +122,14 @@ impl CompressionFormat { Ok(decoder) } - - fn convert_level(level: CompressionLevel) -> Result { - match level { - CompressionLevel::Fastest => Ok(async_compression::Level::Fastest), - CompressionLevel::Best => Ok(async_compression::Level::Best), - CompressionLevel::Default => Ok(async_compression::Level::Default), - CompressionLevel::Level(n) => { - let level = i32::try_from(n).map_err(|_| ZipError::InvalidCompressionLevel(n))?; - Ok(async_compression::Level::Precise(level)) - } - } - } - - pub fn get_encoder(&self, output: W, level: CompressionLevel) -> Result> - where - W: AsyncWrite + Send + Unpin + 'static, - { - let writer = BufWriter::new(output); - - let encoder: Box = match self { - CompressionFormat::Gzip => Box::new(GzipEncoder::with_quality(writer, Self::convert_level(level)?)), - CompressionFormat::Bzip2 => Box::new(BzEncoder::with_quality(writer, Self::convert_level(level)?)), - CompressionFormat::Zlib => Box::new(ZlibEncoder::with_quality(writer, Self::convert_level(level)?)), - CompressionFormat::Xz => Box::new(XzEncoder::with_quality(writer, Self::convert_level(level)?)), - CompressionFormat::Zstd => Box::new(ZstdEncoder::with_quality(writer, Self::convert_level(level)?)), - CompressionFormat::Tar => Box::new(writer), - CompressionFormat::Zip => { - return Err(ZipError::UnsupportedFormat { - format: *self, - operation: "stream encoding", - }); - } - CompressionFormat::Unknown => { - return Err(ZipError::UnsupportedFormat { - format: *self, - operation: "encoding", - }); - } - }; - - Ok(encoder) - } -} - -impl ArchiveFormat { - pub fn from_extension(ext: &str) -> Self { - match ext.to_ascii_lowercase().as_str() { - "gz" | "gzip" | "tgz" => ArchiveFormat::TarGzip, - "bz2" | "bzip2" | "tbz" | "tbz2" => ArchiveFormat::TarBzip2, - "xz" | "txz" => ArchiveFormat::TarXz, - "zlib" | "zz" => ArchiveFormat::TarZlib, - "zst" | "zstd" | "tzst" => ArchiveFormat::TarZstd, - "tar" => ArchiveFormat::Tar, - "zip" => ArchiveFormat::Zip, - _ => ArchiveFormat::Unknown, - } - } - - pub fn from_path>(path: P) -> Self { - let path = path.as_ref(); - let lower_name = path.file_name().and_then(|name| name.to_str()).map(str::to_ascii_lowercase); - - if let Some(name) = lower_name { - if name.ends_with(".tar.gz") || name.ends_with(".tgz") { - return ArchiveFormat::TarGzip; - } - if name.ends_with(".tar.bz2") || name.ends_with(".tbz") || name.ends_with(".tbz2") { - return ArchiveFormat::TarBzip2; - } - if name.ends_with(".tar.xz") || name.ends_with(".txz") { - return ArchiveFormat::TarXz; - } - if name.ends_with(".tar.zst") || name.ends_with(".tzst") { - return ArchiveFormat::TarZstd; - } - if name.ends_with(".tar.zlib") { - return ArchiveFormat::TarZlib; - } - } - - path.extension() - .and_then(|s| s.to_str()) - .map(Self::from_extension) - .unwrap_or(ArchiveFormat::Unknown) - } - - pub fn archive_kind(&self) -> Option { - match self { - ArchiveFormat::Tar - | ArchiveFormat::TarGzip - | ArchiveFormat::TarBzip2 - | ArchiveFormat::TarXz - | ArchiveFormat::TarZlib - | ArchiveFormat::TarZstd => Some(ArchiveKind::Tar), - ArchiveFormat::Zip => Some(ArchiveKind::Zip), - ArchiveFormat::Unknown => None, - } - } - - pub fn compression_codec(&self) -> Option { - match self { - ArchiveFormat::TarGzip => Some(CompressionCodec::Gzip), - ArchiveFormat::TarBzip2 => Some(CompressionCodec::Bzip2), - ArchiveFormat::TarXz => Some(CompressionCodec::Xz), - ArchiveFormat::TarZlib => Some(CompressionCodec::Zlib), - ArchiveFormat::TarZstd => Some(CompressionCodec::Zstd), - ArchiveFormat::Tar | ArchiveFormat::Zip | ArchiveFormat::Unknown => None, - } - } - - pub fn extension(&self) -> &'static str { - match self { - ArchiveFormat::Tar => "tar", - ArchiveFormat::TarGzip => "tar.gz", - ArchiveFormat::TarBzip2 => "tar.bz2", - ArchiveFormat::TarXz => "tar.xz", - ArchiveFormat::TarZlib => "tar.zlib", - ArchiveFormat::TarZstd => "tar.zst", - ArchiveFormat::Zip => "zip", - ArchiveFormat::Unknown => "", - } - } -} - -/// Read entries from a tar-family archive stream. -/// -/// Supported formats are: -/// - `CompressionFormat::Tar` -/// - `CompressionFormat::Gzip` -/// - `CompressionFormat::Bzip2` -/// - `CompressionFormat::Xz` -/// - `CompressionFormat::Zlib` -/// - `CompressionFormat::Zstd` -/// -/// `CompressionFormat::Zip` is intentionally not supported here because ZIP -/// requires central-directory semantics and is handled through file-based -/// helper APIs. -pub async fn read_archive_entries(input: R, format: CompressionFormat, callback: F) -> Result<()> -where - R: AsyncRead + Send + Unpin + 'static, - F: FnMut(tokio_tar::Entry>>) -> Fut + Send + 'static, - Fut: Future> + Send, -{ - read_archive_entries_with_limits(input, format, ArchiveLimits::default(), callback).await -} - -pub async fn read_archive_entries_with_limits( - input: R, - format: CompressionFormat, - limits: ArchiveLimits, - mut callback: F, -) -> Result<()> -where - R: AsyncRead + Send + Unpin + 'static, - F: FnMut(tokio_tar::Entry>>) -> Fut + Send + 'static, - Fut: Future> + Send, -{ - let decoder = format.get_decoder(input)?; - let mut ar = Archive::new(decoder); - let mut entries = ar.entries()?; - let mut entry_count = 0_usize; - let mut total_unpacked_size = 0_u64; - - while let Some(entry) = entries.next().await { - let entry = entry?; - entry_count += 1; - validate_archive_entry_count(entry_count, limits)?; - - let entry_path = entry.path()?.to_string_lossy().into_owned(); - validate_archive_entry_name(&entry_path, limits)?; - - let entry_size = entry.header().size()?; - validate_archive_entry_size(&entry_path, entry_size, limits)?; - total_unpacked_size = total_unpacked_size.saturating_add(entry_size); - validate_archive_total_size(total_unpacked_size, limits)?; - - callback(entry).await?; - } - - Ok(()) -} - -/// Backward-compatible wrapper for archive entry iteration. -pub async fn decompress(input: R, format: CompressionFormat, callback: F) -> Result<()> -where - R: AsyncRead + Send + Unpin + 'static, - F: FnMut(tokio_tar::Entry>>) -> Fut + Send + 'static, - Fut: Future> + Send, -{ - read_archive_entries(input, format, callback).await -} - -/// Explicit tar-family alias for callers that want a clearer name than -/// `decompress()`. -pub async fn extract_tar_entries(input: R, format: CompressionFormat, callback: F) -> Result<()> -where - R: AsyncRead + Send + Unpin + 'static, - F: FnMut(tokio_tar::Entry>>) -> Fut + Send + 'static, - Fut: Future> + Send, -{ - read_archive_entries(input, format, callback).await -} - -fn normalize_zip_entry_name(name: &str) -> Result { - let path = Path::new(name); - let mut normalized = PathBuf::new(); - - for component in path.components() { - match component { - Component::Normal(part) => normalized.push(part), - Component::CurDir => {} - Component::ParentDir | Component::RootDir | Component::Prefix(_) => { - return Err(ZipError::UnsafeEntryPath(name.to_string())); - } - } - } - - let normalized = normalized.to_string_lossy().replace('\\', "/"); - if normalized.is_empty() { - return Err(ZipError::UnsafeEntryPath(name.to_string())); - } - - Ok(normalized) -} - -fn validate_archive_entry_name(name: &str, limits: ArchiveLimits) -> Result<()> { - if !limits.validate_entry_paths { - return Ok(()); - } - - let normalized = normalize_zip_entry_name(name)?; - let length = normalized.len(); - if length > limits.max_path_length { - return Err(ZipError::EntryPathTooLong { - path: normalized, - length, - limit: limits.max_path_length, - }); - } - - Ok(()) -} - -fn validate_archive_entry_size(path: &str, size: u64, limits: ArchiveLimits) -> Result<()> { - if size > limits.max_entry_size { - return Err(ZipError::EntrySizeLimitExceeded { - path: path.to_string(), - size, - limit: limits.max_entry_size, - }); - } - - Ok(()) -} - -fn validate_archive_entry_count(count: usize, limits: ArchiveLimits) -> Result<()> { - if count > limits.max_entries { - return Err(ZipError::EntryCountLimitExceeded { - count, - limit: limits.max_entries, - }); - } - - Ok(()) -} - -fn validate_archive_total_size(total_size: u64, limits: ArchiveLimits) -> Result<()> { - if total_size > limits.max_total_unpacked_size { - return Err(ZipError::TotalUnpackedSizeLimitExceeded { - size: total_size, - limit: limits.max_total_unpacked_size, - }); - } - - Ok(()) -} - -fn zip_method_for_level(level: CompressionLevel) -> CompressionMethod { - match level { - CompressionLevel::Fastest => CompressionMethod::Stored, - CompressionLevel::Best | CompressionLevel::Default | CompressionLevel::Level(_) => CompressionMethod::Deflated, - } -} - -fn parent_directories_for(path: &str) -> Vec { - let path = Path::new(path); - let mut current = PathBuf::new(); - let mut directories = Vec::new(); - - if let Some(parent) = path.parent() { - for component in parent.components() { - if let Component::Normal(part) = component { - current.push(part); - directories.push(format!("{}/", current.to_string_lossy().replace('\\', "/"))); - } - } - } - - directories -} - -fn ensure_directory(path: &Path, created_directories: &mut HashSet) -> Result<()> { - let path = path.to_path_buf(); - if created_directories.insert(path.clone()) { - std::fs::create_dir_all(&path)?; - } - - Ok(()) -} - -fn write_small_zip_entry(reader: &mut R, output_path: &Path, size: u64) -> Result<()> { - let size = usize::try_from(size).map_err(|_| io::Error::other("small zip entry size overflow"))?; - let mut buffer = [0_u8; SMALL_ZIP_EXTRACT_FAST_PATH_LIMIT as usize]; - reader.read_exact(&mut buffer[..size])?; - // `read_exact` stops as soon as the declared bytes are read and never performs the - // terminal zero-length read that the zip crate's `Crc32Reader` uses to validate the - // entry checksum. Force one extra read to EOF so a corrupted small entry is rejected - // here, matching the large-entry `io::copy` path which already reads through EOF. - let mut trailing = [0_u8; 1]; - if reader.read(&mut trailing)? != 0 { - return Err(io::Error::other("small zip entry produced more data than its declared size").into()); - } - std::fs::write(output_path, &buffer[..size])?; - Ok(()) -} - -pub async fn extract_zip_simple, Q: AsRef>(zip_path: P, extract_to: Q) -> Result> { - extract_zip_with_limits(zip_path, extract_to, ArchiveLimits::default()).await -} - -pub async fn extract_zip_to_path_with_limits, Q: AsRef>( - zip_path: P, - extract_to: Q, - limits: ArchiveLimits, -) -> Result { - let zip_path = zip_path.as_ref().to_path_buf(); - let extract_to = extract_to.as_ref().to_path_buf(); - - spawn_blocking(move || extract_zip_impl(zip_path, extract_to, limits, false).map(|(_, summary)| summary)).await? -} - -pub async fn extract_zip_with_limits, Q: AsRef>( - zip_path: P, - extract_to: Q, - limits: ArchiveLimits, -) -> Result> { - let zip_path = zip_path.as_ref().to_path_buf(); - let extract_to = extract_to.as_ref().to_path_buf(); - - spawn_blocking(move || extract_zip_impl(zip_path, extract_to, limits, true).map(|(entries, _)| entries.unwrap_or_default())) - .await? -} - -fn extract_zip_impl( - zip_path: PathBuf, - extract_to: PathBuf, - limits: ArchiveLimits, - collect_entries: bool, -) -> Result<(Option>, ZipExtractSummary)> { - let file = std::fs::File::open(&zip_path)?; - let mut archive = ZipArchive::new(file)?; - std::fs::create_dir_all(&extract_to)?; - let mut created_directories = HashSet::from([extract_to.clone()]); - - let mut entries = collect_entries.then(|| Vec::with_capacity(archive.len())); - let mut summary = ZipExtractSummary::default(); - for index in 0..archive.len() { - validate_archive_entry_count(index + 1, limits)?; - let mut zip_file = archive.by_index(index)?; - let enclosed_name = zip_file - .enclosed_name() - .ok_or_else(|| ZipError::UnsafeEntryPath(zip_file.name().to_string()))?; - let entry_name = enclosed_name.to_string_lossy().replace('\\', "/"); - let is_dir = zip_file.is_dir(); - let size = zip_file.size(); - validate_archive_entry_name(&entry_name, limits)?; - validate_archive_entry_size(&entry_name, size, limits)?; - summary.total_unpacked_size = summary.total_unpacked_size.saturating_add(size); - validate_archive_total_size(summary.total_unpacked_size, limits)?; - let output_path = extract_to.join(&enclosed_name); - - if is_dir { - ensure_directory(&output_path, &mut created_directories)?; - summary.directory_count += 1; - } else { - if let Some(parent) = output_path.parent() { - ensure_directory(parent, &mut created_directories)?; - } - if size <= SMALL_ZIP_EXTRACT_FAST_PATH_LIMIT { - write_small_zip_entry(&mut zip_file, &output_path, size)?; - } else { - let mut output = std::fs::File::create(&output_path)?; - std::io::copy(&mut zip_file, &mut output)?; - } - summary.file_count += 1; - } - summary.entry_count += 1; - - if let Some(ref mut entries) = entries { - entries.push(ZipEntry { - name: entry_name, - size, - compressed_size: zip_file.compressed_size(), - is_dir, - compression_method: format!("{:?}", zip_file.compression()), - archive_kind: ArchiveKind::Zip, - format: ArchiveFormat::Zip, - unix_mode: zip_file.unix_mode(), - }); - } - } - - Ok((entries, summary)) -} - -pub async fn create_zip_simple>( - zip_path: P, - files: Vec<(String, Vec)>, - compression_level: CompressionLevel, -) -> Result<()> { - create_zip_with_options( - zip_path, - files, - ZipWriteOptions { - compression_level, - ..ZipWriteOptions::default() - }, - ) - .await -} - -pub async fn create_zip_with_options>( - zip_path: P, - files: Vec<(String, Vec)>, - options: ZipWriteOptions, -) -> Result<()> { - let zip_path = zip_path.as_ref().to_path_buf(); - - spawn_blocking(move || -> Result<()> { - if let Some(parent) = zip_path.parent() { - std::fs::create_dir_all(parent)?; - } - - let file = std::fs::File::create(&zip_path)?; - let mut writer = ZipWriter::new(file); - let file_options = SimpleFileOptions::default().compression_method(zip_method_for_level(options.compression_level)); - let explicit_directories = files - .iter() - .filter(|(name, _)| name.ends_with('/')) - .map(|(name, _)| normalize_zip_entry_name(name)) - .collect::>>()?; - let mut written_directories = HashSet::new(); - - for (name, contents) in files { - let entry_name = normalize_zip_entry_name(&name)?; - if name.ends_with('/') { - if written_directories.insert(entry_name.clone()) { - writer.add_directory(entry_name, file_options)?; - } - } else { - if options.create_directory_entries { - for directory in parent_directories_for(&entry_name) { - if !explicit_directories.contains(&directory) && written_directories.insert(directory.clone()) { - writer.add_directory(directory, file_options)?; - } - } - } - writer.start_file(entry_name, file_options)?; - writer.write_all(&contents)?; - } - } - - writer.finish()?; - Ok(()) - }) - .await? -} - -pub struct Compressor { - format: CompressionFormat, - level: CompressionLevel, -} - -impl Compressor { - pub fn new(format: CompressionFormat) -> Self { - Self { - format, - level: CompressionLevel::Default, - } - } - - pub fn with_level(mut self, level: CompressionLevel) -> Self { - self.level = level; - self - } - - pub async fn compress(&self, input: &[u8]) -> Result> { - let sink = SharedBuffer::default(); - let mut encoder = self.format.get_encoder(sink.clone(), self.level)?; - let mut reader = input; - - io::copy(&mut reader, &mut encoder).await?; - encoder.shutdown().await?; - drop(encoder); - - Ok(sink.into_vec()) - } - - pub async fn decompress(&self, input: Vec) -> Result> { - let mut output = Vec::new(); - let cursor = std::io::Cursor::new(input); - let mut decoder = self.format.get_decoder(cursor)?; - - decoder.read_to_end(&mut output).await?; - Ok(output) - } -} - -pub struct Decompressor { - format: CompressionFormat, -} - -impl Decompressor { - pub fn new(format: CompressionFormat) -> Self { - Self { format } - } - - pub fn auto_detect>(path: P) -> Self { - Self { - format: CompressionFormat::from_path(path), - } - } - - pub async fn decompress_file>(&self, input_path: P, output_path: P) -> Result<()> { - let input_file = File::open(&input_path).await?; - let output_file = File::create(&output_path).await?; - - let mut decoder = self.format.get_decoder(input_file)?; - let mut writer = BufWriter::new(output_file); - - io::copy(&mut decoder, &mut writer).await?; - writer.shutdown().await?; - - Ok(()) - } } #[cfg(test)] mod tests { use super::*; + use async_compression::tokio::write::GzipEncoder; use std::mem::size_of; - use tempfile::tempdir; - use tokio::fs; - use tokio::io::AsyncReadExt; - use tokio_tar::{Builder, Header}; - use zip::write::FileOptions; - - async fn build_tar_bytes(files: &[(&str, &[u8])]) -> io::Result> { - let sink = SharedBuffer::default(); - let handle = sink.clone(); - let mut builder = Builder::new(sink); - - for (path, content) in files { - let mut header = Header::new_gnu(); - header.set_size(content.len() as u64); - header.set_mode(0o644); - header.set_cksum(); - builder.append_data(&mut header, *path, &content[..]).await?; - } - - builder.finish().await?; - Ok(handle.into_vec()) - } - - async fn build_compressed_tar_bytes(format: CompressionFormat, files: &[(&str, &[u8])]) -> Result> { - let tar_bytes = build_tar_bytes(files).await?; - Compressor::new(format).compress(&tar_bytes).await - } - - async fn build_zip_file_with_entries(path: &Path, files: &[(&str, &[u8])]) -> Result<()> { - let path = path.to_path_buf(); - let files = files - .iter() - .map(|(name, content)| ((*name).to_string(), content.to_vec())) - .collect::>(); - spawn_blocking(move || -> Result<()> { - let file = std::fs::File::create(path)?; - let mut writer = ZipWriter::new(file); - let options: FileOptions<'_, ()> = FileOptions::default().compression_method(CompressionMethod::Stored); - for (name, content) in files { - writer.start_file(name, options)?; - writer.write_all(&content)?; - } - writer.finish()?; - Ok(()) - }) - .await??; - Ok(()) - } - - async fn collect_archive_entries(payload: Vec, format: CompressionFormat) -> Result)>> { - let seen = Arc::new(Mutex::new(Vec::<(String, Vec)>::new())); - let seen_ref = Arc::clone(&seen); - let cursor = std::io::Cursor::new(payload); - - read_archive_entries(cursor, format, move |mut entry| { - let seen_ref = Arc::clone(&seen_ref); - async move { - let path = entry.path()?.to_string_lossy().into_owned(); - let mut content = Vec::new(); - entry.read_to_end(&mut content).await?; - seen_ref.lock().expect("seen collection lock poisoned").push((path, content)); - Ok(()) - } - }) - .await?; - - Ok(seen.lock().expect("seen collection lock poisoned").clone()) - } + use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[test] fn test_compression_format_from_extension() { @@ -918,869 +139,51 @@ mod tests { assert_eq!(CompressionFormat::from_extension("txt"), CompressionFormat::Unknown); } - #[test] - fn test_archive_format_from_extension() { - assert_eq!(ArchiveFormat::from_extension("gz"), ArchiveFormat::TarGzip); - assert_eq!(ArchiveFormat::from_extension("tbz2"), ArchiveFormat::TarBzip2); - assert_eq!(ArchiveFormat::from_extension("txz"), ArchiveFormat::TarXz); - assert_eq!(ArchiveFormat::from_extension("zip"), ArchiveFormat::Zip); - assert_eq!(ArchiveFormat::from_extension("txt"), ArchiveFormat::Unknown); - } - - #[test] - fn test_compression_format_from_path_handles_compound_suffixes() { - assert_eq!(CompressionFormat::from_path("archive.tar.gz"), CompressionFormat::Gzip); - assert_eq!(CompressionFormat::from_path("archive.tgz"), CompressionFormat::Gzip); - assert_eq!(CompressionFormat::from_path("archive.tar.bz2"), CompressionFormat::Bzip2); - assert_eq!(CompressionFormat::from_path("archive.zip"), CompressionFormat::Zip); - assert_eq!(CompressionFormat::from_path("archive"), CompressionFormat::Unknown); - } - - #[test] - fn test_archive_format_from_path_handles_compound_suffixes() { - assert_eq!(ArchiveFormat::from_path("archive.tar.gz"), ArchiveFormat::TarGzip); - assert_eq!(ArchiveFormat::from_path("archive.tar.bz2"), ArchiveFormat::TarBzip2); - assert_eq!(ArchiveFormat::from_path("archive.tar.xz"), ArchiveFormat::TarXz); - assert_eq!(ArchiveFormat::from_path("archive.tar.zst"), ArchiveFormat::TarZstd); - assert_eq!(ArchiveFormat::from_path("archive.zip"), ArchiveFormat::Zip); - assert_eq!(ArchiveFormat::from_path("archive"), ArchiveFormat::Unknown); - } - - #[test] - fn test_archive_format_and_legacy_compression_format_are_compatible() { - assert_eq!(CompressionFormat::from_archive_format(ArchiveFormat::TarGzip), CompressionFormat::Gzip); - assert_eq!(CompressionFormat::from_archive_format(ArchiveFormat::Tar), CompressionFormat::Tar); - assert_eq!(CompressionFormat::from_archive_format(ArchiveFormat::Zip), CompressionFormat::Zip); - } - - #[test] - fn test_archive_format_exposes_archive_kind_and_codec() { - assert_eq!(ArchiveFormat::TarGzip.archive_kind(), Some(ArchiveKind::Tar)); - assert_eq!(ArchiveFormat::TarGzip.compression_codec(), Some(CompressionCodec::Gzip)); - assert_eq!(ArchiveFormat::Tar.archive_kind(), Some(ArchiveKind::Tar)); - assert_eq!(ArchiveFormat::Tar.compression_codec(), None); - assert_eq!(ArchiveFormat::Zip.archive_kind(), Some(ArchiveKind::Zip)); - assert_eq!(ArchiveFormat::Zip.compression_codec(), None); - } - - #[test] - fn test_legacy_compression_format_exposes_kind_and_codec() { - assert_eq!(CompressionFormat::Gzip.archive_kind(), None); - assert_eq!(CompressionFormat::Gzip.compression_codec(), Some(CompressionCodec::Gzip)); - assert_eq!(CompressionFormat::Tar.archive_kind(), Some(ArchiveKind::Tar)); - assert_eq!(CompressionFormat::Tar.compression_codec(), None); - assert_eq!(CompressionFormat::Zip.archive_kind(), Some(ArchiveKind::Zip)); - } - #[test] fn test_compression_format_size_is_small() { assert!(size_of::() <= 8); assert!(size_of::>() <= 16); } - #[test] - fn test_convert_level_rejects_overflow() { - let err = match CompressionFormat::Gzip.get_encoder(SharedBuffer::default(), CompressionLevel::Level(u32::MAX)) { - Ok(_) => panic!("overflow level should return an error"), - Err(err) => err, - }; - assert!(matches!(err, ZipError::InvalidCompressionLevel(u32::MAX))); - } + #[tokio::test] + async fn test_get_decoder_round_trips_gzip_stream() { + let mut encoder = GzipEncoder::new(Vec::new()); + encoder.write_all(b"payload").await.expect("gzip encode should succeed"); + encoder.shutdown().await.expect("gzip encoder shutdown should succeed"); - #[test] - fn test_validate_archive_entry_name_rejects_absolute_path() { - let err = validate_archive_entry_name("/absolute.txt", ArchiveLimits::default()) - .expect_err("absolute path should fail validation"); - assert!(matches!(err, ZipError::UnsafeEntryPath(path) if path == "/absolute.txt")); + let mut decoder = CompressionFormat::Gzip + .get_decoder(std::io::Cursor::new(encoder.into_inner())) + .expect("gzip decoder should be created"); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).await.expect("gzip decode should succeed"); + + assert_eq!(decoded, b"payload"); } #[tokio::test] - async fn test_compressor_round_trip_gzip() { - let input = b"hello rustfs zip ".repeat(64); - let compressor = Compressor::new(CompressionFormat::Gzip); - - let compressed = compressor.compress(&input).await.expect("gzip compress should succeed"); - assert!(!compressed.is_empty()); - assert_ne!(compressed, input); - - let decompressed = compressor - .decompress(compressed) - .await - .expect("gzip decompress should succeed"); - assert_eq!(decompressed, input); - } - - #[tokio::test] - async fn test_compressor_round_trip_zstd() { - let input = b"zstd payload ".repeat(128); - let compressor = Compressor::new(CompressionFormat::Zstd).with_level(CompressionLevel::Best); - - let compressed = compressor.compress(&input).await.expect("zstd compress should succeed"); - let decompressed = compressor - .decompress(compressed) - .await - .expect("zstd decompress should succeed"); - assert_eq!(decompressed, input); - } - - #[tokio::test] - async fn test_zip_stream_encoder_is_rejected() { - let err = CompressionFormat::Zip - .get_encoder(SharedBuffer::default(), CompressionLevel::Default) + async fn test_get_decoder_rejects_zip_and_unknown_formats() { + let zip_err = CompressionFormat::Zip + .get_decoder(std::io::Cursor::new(Vec::::new())) .err() - .expect("zip stream encoder should be rejected"); + .expect("zip stream decoding should be rejected"); assert!(matches!( - err, - ZipError::UnsupportedFormat { - format: CompressionFormat::Zip, - operation: "stream encoding", - } - )); - } - - #[tokio::test] - async fn test_read_archive_entries_iterates_tar_gzip_entries() { - let gzip_bytes = - build_compressed_tar_bytes(CompressionFormat::Gzip, &[("nested/hello.txt", b"hello"), ("world.txt", b"world")]) - .await - .expect("tar.gz build should succeed"); - - let seen = collect_archive_entries(gzip_bytes, CompressionFormat::Gzip) - .await - .expect("tar.gz archive iteration should succeed"); - assert_eq!(seen.len(), 2); - assert_eq!(seen[0].0, "nested/hello.txt"); - assert_eq!(seen[0].1, b"hello"); - assert_eq!(seen[1].0, "world.txt"); - assert_eq!(seen[1].1, b"world"); - } - - #[tokio::test] - async fn test_read_archive_entries_iterates_tar_bzip2_entries() { - let payload = - build_compressed_tar_bytes(CompressionFormat::Bzip2, &[("nested/hello.txt", b"hello"), ("world.txt", b"world")]) - .await - .expect("tar.bz2 build should succeed"); - - let seen = collect_archive_entries(payload, CompressionFormat::Bzip2) - .await - .expect("tar.bz2 archive iteration should succeed"); - assert_eq!(seen.len(), 2); - assert_eq!(seen[0].0, "nested/hello.txt"); - assert_eq!(seen[1].0, "world.txt"); - } - - #[tokio::test] - async fn test_read_archive_entries_iterates_tar_xz_entries() { - let payload = - build_compressed_tar_bytes(CompressionFormat::Xz, &[("nested/hello.txt", b"hello"), ("world.txt", b"world")]) - .await - .expect("tar.xz build should succeed"); - - let seen = collect_archive_entries(payload, CompressionFormat::Xz) - .await - .expect("tar.xz archive iteration should succeed"); - assert_eq!(seen.len(), 2); - assert_eq!(seen[0].0, "nested/hello.txt"); - assert_eq!(seen[1].0, "world.txt"); - } - - #[tokio::test] - async fn test_read_archive_entries_iterates_tar_zstd_entries() { - let payload = - build_compressed_tar_bytes(CompressionFormat::Zstd, &[("nested/hello.txt", b"hello"), ("world.txt", b"world")]) - .await - .expect("tar.zst build should succeed"); - - let seen = collect_archive_entries(payload, CompressionFormat::Zstd) - .await - .expect("tar.zst archive iteration should succeed"); - assert_eq!(seen.len(), 2); - assert_eq!(seen[0].0, "nested/hello.txt"); - assert_eq!(seen[1].0, "world.txt"); - } - - #[tokio::test] - async fn test_extract_tar_entries_alias_matches_stream_behavior() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("hello.txt", b"hello")]) - .await - .expect("tar.gz build should succeed"); - let seen = Arc::new(Mutex::new(Vec::::new())); - let seen_ref = Arc::clone(&seen); - - extract_tar_entries(std::io::Cursor::new(payload), CompressionFormat::Gzip, move |entry| { - let seen_ref = Arc::clone(&seen_ref); - async move { - seen_ref - .lock() - .expect("seen collection lock poisoned") - .push(entry.path()?.to_string_lossy().into_owned()); - Ok(()) - } - }) - .await - .expect("extract_tar_entries alias should succeed"); - - assert_eq!(seen.lock().expect("seen collection lock poisoned").as_slice(), ["hello.txt"]); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_zip_streams() { - let err = read_archive_entries(std::io::Cursor::new(Vec::::new()), CompressionFormat::Zip, |_entry| async { Ok(()) }) - .await - .expect_err("zip stream should be rejected"); - - assert!(matches!( - err, + zip_err, ZipError::UnsupportedFormat { format: CompressionFormat::Zip, operation: "stream decoding", } )); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_corrupt_tar_gzip_stream() { - let err = read_archive_entries( - std::io::Cursor::new(b"not-a-valid-gzip-stream".to_vec()), - CompressionFormat::Gzip, - |_entry| async { Ok(()) }, - ) - .await - .expect_err("corrupt tar.gz stream should fail"); - - assert!(matches!(err, ZipError::Io(_))); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_truncated_tar_gzip_stream() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("hello.txt", b"hello world")]) - .await - .expect("tar.gz build should succeed"); - let truncated = payload[..payload.len() / 2].to_vec(); - - let err = read_archive_entries(std::io::Cursor::new(truncated), CompressionFormat::Gzip, |_entry| async { Ok(()) }) - .await - .expect_err("truncated tar.gz stream should fail"); - - assert!(matches!(err, ZipError::Io(_))); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_too_many_entries() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("one.txt", b"1"), ("two.txt", b"2")]) - .await - .expect("tar.gz build should succeed"); - - let err = read_archive_entries_with_limits( - std::io::Cursor::new(payload), - CompressionFormat::Gzip, - ArchiveLimits { - max_entries: 1, - ..ArchiveLimits::default() - }, - |_entry| async { Ok(()) }, - ) - .await - .expect_err("entry count limit should fail"); - - assert!(matches!(err, ZipError::EntryCountLimitExceeded { count: 2, limit: 1 })); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_oversized_entry() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("big.txt", b"hello world")]) - .await - .expect("tar.gz build should succeed"); - - let err = read_archive_entries_with_limits( - std::io::Cursor::new(payload), - CompressionFormat::Gzip, - ArchiveLimits { - max_entry_size: 4, - ..ArchiveLimits::default() - }, - |_entry| async { Ok(()) }, - ) - .await - .expect_err("entry size limit should fail"); + let unknown_err = CompressionFormat::Unknown + .get_decoder(std::io::Cursor::new(Vec::::new())) + .err() + .expect("unknown format decoding should be rejected"); assert!(matches!( - err, - ZipError::EntrySizeLimitExceeded { - path, - size: 11, - limit: 4, - } if path == "big.txt" + unknown_err, + ZipError::UnsupportedFormat { + format: CompressionFormat::Unknown, + operation: "decoding", + } )); } - - #[tokio::test] - async fn test_read_archive_entries_rejects_total_unpacked_size_limit() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("one.txt", b"12345"), ("two.txt", b"67890")]) - .await - .expect("tar.gz build should succeed"); - - let err = read_archive_entries_with_limits( - std::io::Cursor::new(payload), - CompressionFormat::Gzip, - ArchiveLimits { - max_total_unpacked_size: 9, - ..ArchiveLimits::default() - }, - |_entry| async { Ok(()) }, - ) - .await - .expect_err("total unpacked size limit should fail"); - - assert!(matches!(err, ZipError::TotalUnpackedSizeLimitExceeded { size: 10, limit: 9 })); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_entry_path_length_limit() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("nested/hello.txt", b"hello")]) - .await - .expect("tar.gz build should succeed"); - - let err = read_archive_entries_with_limits( - std::io::Cursor::new(payload), - CompressionFormat::Gzip, - ArchiveLimits { - max_path_length: 5, - ..ArchiveLimits::default() - }, - |_entry| async { Ok(()) }, - ) - .await - .expect_err("path length limit should fail"); - - assert!(matches!( - err, - ZipError::EntryPathTooLong { path, limit: 5, .. } if path == "nested/hello.txt" - )); - } - - #[tokio::test] - async fn test_create_and_extract_zip_round_trip() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - - create_zip_simple( - &zip_path, - vec![ - ("nested/hello.txt".to_string(), b"hello".to_vec()), - ("world.txt".to_string(), b"world".to_vec()), - ], - CompressionLevel::Default, - ) - .await - .expect("zip creation should succeed"); - - let entries = extract_zip_simple(&zip_path, &extract_path) - .await - .expect("zip extraction should succeed"); - assert_eq!(entries.len(), 2); - assert_eq!( - fs::read(extract_path.join("nested/hello.txt")) - .await - .expect("nested zip entry should be extracted"), - b"hello" - ); - assert_eq!( - fs::read(extract_path.join("world.txt")) - .await - .expect("root zip entry should be extracted"), - b"world" - ); - assert!(entries.iter().all(|entry| entry.archive_kind == ArchiveKind::Zip)); - assert!(entries.iter().all(|entry| entry.format == ArchiveFormat::Zip)); - } - - #[tokio::test] - async fn test_create_zip_with_directory_entries_and_extract_directory_scenarios() { - let temp = tempdir().expect("tempdir should be created"); - let explicit_zip_path = temp.path().join("explicit-directories.zip"); - let explicit_extract_path = temp.path().join("explicit-extract"); - let auto_zip_path = temp.path().join("auto-directories.zip"); - let auto_extract_path = temp.path().join("auto-extract"); - - create_zip_with_options( - &explicit_zip_path, - vec![ - ("nested/".to_string(), Vec::new()), - ("nested/deeper/".to_string(), Vec::new()), - ], - ZipWriteOptions { - compression_level: CompressionLevel::Default, - create_directory_entries: false, - }, - ) - .await - .expect("zip creation with explicit directory entries should succeed"); - - let explicit_entries = extract_zip_with_limits(&explicit_zip_path, &explicit_extract_path, ArchiveLimits::default()) - .await - .expect("zip extraction with explicit directory entries should succeed"); - - assert!( - explicit_entries - .iter() - .any(|entry| entry.name.trim_end_matches('/') == "nested" && entry.is_dir) - ); - assert!( - explicit_entries - .iter() - .any(|entry| entry.name.trim_end_matches('/') == "nested/deeper" && entry.is_dir) - ); - assert!( - fs::metadata(explicit_extract_path.join("nested")) - .await - .expect("nested directory should exist") - .is_dir() - ); - assert!( - fs::metadata(explicit_extract_path.join("nested/deeper")) - .await - .expect("nested deeper directory should exist") - .is_dir() - ); - - create_zip_with_options( - &auto_zip_path, - vec![("nested/deeper/file.txt".to_string(), b"hello".to_vec())], - ZipWriteOptions { - compression_level: CompressionLevel::Default, - create_directory_entries: true, - }, - ) - .await - .expect("zip creation with automatic directory entries should succeed"); - - let entries = extract_zip_with_limits(&auto_zip_path, &auto_extract_path, ArchiveLimits::default()) - .await - .expect("zip extraction with automatic directory entries should succeed"); - - assert!( - entries - .iter() - .any(|entry| entry.name.trim_end_matches('/') == "nested" && entry.is_dir) - ); - assert!( - entries - .iter() - .any(|entry| entry.name.trim_end_matches('/') == "nested/deeper" && entry.is_dir) - ); - assert!( - fs::metadata(auto_extract_path.join("nested")) - .await - .expect("nested directory should exist") - .is_dir() - ); - assert!( - fs::metadata(auto_extract_path.join("nested/deeper")) - .await - .expect("nested deeper directory should exist") - .is_dir() - ); - assert_eq!( - fs::read(auto_extract_path.join("nested/deeper/file.txt")) - .await - .expect("nested file should be extracted"), - b"hello" - ); - } - - #[tokio::test] - async fn test_create_zip_with_lots_of_small_files_round_trip() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("many-small-files.zip"); - let extract_path = temp.path().join("extract"); - let files = (0..32) - .map(|index| (format!("batch/file-{index}.txt"), format!("payload-{index}").into_bytes())) - .collect::>(); - - create_zip_with_options( - &zip_path, - files.clone(), - ZipWriteOptions { - compression_level: CompressionLevel::Default, - create_directory_entries: true, - }, - ) - .await - .expect("zip creation for many small files should succeed"); - - let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("zip extraction for many small files should succeed"); - - assert!(entries.len() >= files.len()); - for (path, expected) in files { - assert_eq!( - fs::read(extract_path.join(path)) - .await - .expect("small file should be extracted"), - expected - ); - } - } - - #[tokio::test] - async fn test_extract_zip_to_path_with_limits_returns_summary() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("summary.zip"); - let extract_path = temp.path().join("extract"); - - create_zip_with_options( - &zip_path, - vec![ - ("nested/".to_string(), Vec::new()), - ("nested/hello.txt".to_string(), b"hello".to_vec()), - ("world.txt".to_string(), b"world".to_vec()), - ], - ZipWriteOptions { - compression_level: CompressionLevel::Default, - create_directory_entries: false, - }, - ) - .await - .expect("zip creation for summary should succeed"); - - let summary = extract_zip_to_path_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("zip extract summary should succeed"); - - assert_eq!(summary.entry_count, 3); - assert_eq!(summary.directory_count, 1); - assert_eq!(summary.file_count, 2); - assert_eq!(summary.total_unpacked_size, 10); - assert_eq!( - fs::read(extract_path.join("nested/hello.txt")) - .await - .expect("nested file should be extracted"), - b"hello" - ); - assert_eq!( - fs::read(extract_path.join("world.txt")) - .await - .expect("world file should be extracted"), - b"world" - ); - } - - #[tokio::test] - async fn test_zip_helper_exposes_stored_vs_deflated_metadata() { - let temp = tempdir().expect("tempdir should be created"); - let stored_zip_path = temp.path().join("stored.zip"); - let deflated_zip_path = temp.path().join("deflated.zip"); - let stored_extract_path = temp.path().join("stored-extract"); - let deflated_extract_path = temp.path().join("deflated-extract"); - let payload = b"compressible-content-".repeat(64); - - create_zip_with_options( - &stored_zip_path, - vec![("payload.txt".to_string(), payload.clone())], - ZipWriteOptions { - compression_level: CompressionLevel::Fastest, - create_directory_entries: false, - }, - ) - .await - .expect("stored zip creation should succeed"); - - create_zip_with_options( - &deflated_zip_path, - vec![("payload.txt".to_string(), payload.clone())], - ZipWriteOptions { - compression_level: CompressionLevel::Best, - create_directory_entries: false, - }, - ) - .await - .expect("deflated zip creation should succeed"); - - let stored_entries = extract_zip_with_limits(&stored_zip_path, &stored_extract_path, ArchiveLimits::default()) - .await - .expect("stored zip extraction should succeed"); - let deflated_entries = extract_zip_with_limits(&deflated_zip_path, &deflated_extract_path, ArchiveLimits::default()) - .await - .expect("deflated zip extraction should succeed"); - - assert_eq!(stored_entries[0].compression_method, "Stored"); - assert_eq!(deflated_entries[0].compression_method, "Deflated"); - assert_eq!( - fs::read(stored_extract_path.join("payload.txt")) - .await - .expect("stored payload should be extracted"), - payload - ); - assert_eq!( - fs::read(deflated_extract_path.join("payload.txt")) - .await - .expect("deflated payload should be extracted"), - payload - ); - assert!(deflated_entries[0].compressed_size <= stored_entries[0].compressed_size); - } - - #[tokio::test] - async fn test_create_zip_rejects_unsafe_entry_name() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("archive.zip"); - - let err = create_zip_simple( - &zip_path, - vec![("../escape.txt".to_string(), b"escape".to_vec())], - CompressionLevel::Default, - ) - .await - .unwrap_err(); - - assert!(matches!(err, ZipError::UnsafeEntryPath(path) if path == "../escape.txt")); - } - - #[tokio::test] - async fn test_extract_zip_rejects_parent_traversal_entry_and_writes_nothing_outside_target() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("traversal.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("safe.txt", b"safe"), ("../escape.txt", b"escape")]) - .await - .expect("zip fixture should be created"); - - let err = extract_zip_simple(&zip_path, &extract_path) - .await - .expect_err("zip entry with parent traversal should be rejected"); - - assert!(matches!(err, ZipError::UnsafeEntryPath(path) if path == "../escape.txt")); - assert!( - fs::metadata(temp.path().join("escape.txt")).await.is_err(), - "traversal entry must not be written outside the extraction target" - ); - assert!( - fs::metadata(extract_path.join("escape.txt")).await.is_err(), - "traversal entry must not be written inside the extraction target either" - ); - } - - #[tokio::test] - async fn test_extract_zip_rejects_small_entry_with_corrupted_crc() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("corrupt-crc.zip"); - let extract_path = temp.path().join("extract"); - - // A stored entry well under the small-entry fast-path limit so extraction takes - // the in-memory buffer path rather than the streaming `io::copy` path. - let content = b"small-entry-crc-payload"; - assert!((content.len() as u64) <= SMALL_ZIP_EXTRACT_FAST_PATH_LIMIT); - build_zip_file_with_entries(&zip_path, &[("small.txt", content)]) - .await - .expect("zip fixture should be created"); - - // Corrupt the stored entry data so its bytes no longer match the recorded CRC32. - let mut raw = fs::read(&zip_path).await.expect("zip fixture should be readable"); - let offset = raw - .windows(content.len()) - .position(|window| window == content) - .expect("stored entry data should be present in the zip"); - raw[offset] ^= 0xFF; - fs::write(&zip_path, &raw).await.expect("corrupted zip should be writable"); - - let err = extract_zip_simple(&zip_path, &extract_path) - .await - .expect_err("small entry with a corrupted CRC should be rejected"); - - assert!( - matches!(err, ZipError::Io(ref e) if e.kind() == std::io::ErrorKind::InvalidData), - "expected an InvalidData checksum error, got {err:?}" - ); - } - - #[tokio::test] - async fn test_extract_zip_allows_entry_count_exactly_at_limit() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("at-limit.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("one.txt", b"1"), ("two.txt", b"2")]) - .await - .expect("zip fixture should be created"); - - let entries = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_entries: 2, - ..ArchiveLimits::default() - }, - ) - .await - .expect("entry count exactly at the limit should extract successfully"); - - assert_eq!(entries.len(), 2); - } - - #[tokio::test] - async fn test_extract_zip_allows_total_unpacked_size_exactly_at_limit() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("total-at-limit.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("one.txt", b"12345"), ("two.txt", b"67890")]) - .await - .expect("zip fixture should be created"); - - let entries = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_total_unpacked_size: 10, - ..ArchiveLimits::default() - }, - ) - .await - .expect("total unpacked size exactly at the limit should extract successfully"); - - assert_eq!(entries.len(), 2); - assert_eq!( - fs::read(extract_path.join("two.txt")) - .await - .expect("second entry should be extracted"), - b"67890" - ); - } - - #[tokio::test] - async fn test_extract_zip_rejects_too_many_entries() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("too-many.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("one.txt", b"1"), ("two.txt", b"2")]) - .await - .expect("zip fixture should be created"); - - let err = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_entries: 1, - ..ArchiveLimits::default() - }, - ) - .await - .expect_err("zip entry count limit should fail"); - - assert!(matches!(err, ZipError::EntryCountLimitExceeded { count: 2, limit: 1 })); - } - - #[tokio::test] - async fn test_extract_zip_rejects_oversized_entry() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("oversized.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("big.txt", b"hello world")]) - .await - .expect("zip fixture should be created"); - - let err = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_entry_size: 4, - ..ArchiveLimits::default() - }, - ) - .await - .expect_err("zip entry size limit should fail"); - - assert!(matches!( - err, - ZipError::EntrySizeLimitExceeded { - path, - size: 11, - limit: 4, - } if path == "big.txt" - )); - } - - #[tokio::test] - async fn test_extract_zip_rejects_total_unpacked_size_limit() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("too-large-total.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("one.txt", b"12345"), ("two.txt", b"67890")]) - .await - .expect("zip fixture should be created"); - - let err = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_total_unpacked_size: 9, - ..ArchiveLimits::default() - }, - ) - .await - .expect_err("zip total size limit should fail"); - - assert!(matches!(err, ZipError::TotalUnpackedSizeLimitExceeded { size: 10, limit: 9 })); - } - - #[tokio::test] - async fn test_extract_zip_rejects_entry_path_length_limit() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("long-path.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("nested/hello.txt", b"hello")]) - .await - .expect("zip fixture should be created"); - - let err = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_path_length: 5, - ..ArchiveLimits::default() - }, - ) - .await - .expect_err("zip path length limit should fail"); - - assert!(matches!( - err, - ZipError::EntryPathTooLong { path, limit: 5, .. } if path == "nested/hello.txt" - )); - } - - #[tokio::test] - async fn test_decompress_file_round_trip() { - let temp = tempdir().expect("tempdir should be created"); - let input_path = temp.path().join("payload.txt.gz"); - let output_path = temp.path().join("payload.txt"); - let compressed = Compressor::new(CompressionFormat::Gzip) - .compress(b"payload") - .await - .expect("gzip compress should succeed"); - fs::write(&input_path, compressed) - .await - .expect("compressed input file should be written"); - - Decompressor::auto_detect(&input_path) - .decompress_file(&input_path, &output_path) - .await - .expect("gzip file decompress should succeed"); - - assert_eq!( - fs::read(&output_path) - .await - .expect("decompressed output file should be readable"), - b"payload" - ); - } }