gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 21:26:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(e2e): fail closed on compression disk probes (#6492)

Browse Source
This commit is contained in:
Zhengchao An
2026-08-24 14:32:02 +08:00
committed by GitHub
parent 9681f19bec
commit f4ce1a8b3a
1 changed files with 59 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/compression_test.rs
+59 -29
View File
@@ -4,7 +4,8 @@ use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use std::fs;
use std::path::PathBuf;
use std::io;
use std::path::{Path, PathBuf};
use std::process::Command;
use tracing::info;
@@ -31,30 +32,58 @@ fn generate_high_ratio_binary_data(size: usize, seed: u8) -> Vec<u8> {
.collect()
}
fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec<PathBuf> {
fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> io::Result<Vec<PathBuf>> {
let bucket_path = PathBuf::from(temp_dir).join(bucket);
let mut part_files = Vec::new();
fn scan_dir(dir: &PathBuf, target: &str, results: &mut Vec<PathBuf>) {
if let Ok(entries) = fs::read_dir(dir) {
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
scan_dir(&path, target, results);
} else if path
.file_name()
.map(|n| n.to_string_lossy().starts_with("part."))
.unwrap_or(false)
&& path.to_string_lossy().contains(target)
{
results.push(path);
fn scan_dir(dir: &Path, target: &str, results: &mut Vec<PathBuf>) -> io::Result<()> {
let entries = fs::read_dir(dir)
.map_err(|error| io::Error::new(error.kind(), format!("failed to read {}: {error}", dir.display())))?;
for entry in entries {
let entry = entry
.map_err(|error| io::Error::new(error.kind(), format!("failed to read entry in {}: {error}", dir.display())))?;
let path = entry.path();
let file_type = entry
.file_type()
.map_err(|error| io::Error::new(error.kind(), format!("failed to inspect {}: {error}", path.display())))?;
if file_type.is_dir() {
scan_dir(&path, target, results)?;
} else if path
.file_name()
.map(|n| n.to_string_lossy().starts_with("part."))
.unwrap_or(false)
&& path.to_string_lossy().contains(target)
{
if !file_type.is_file() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("expected regular part file at {}", path.display()),
));
}
results.push(path);
}
}
Ok(())
}
scan_dir(&bucket_path, object_key, &mut part_files);
part_files
scan_dir(&bucket_path, object_key, &mut part_files)?;
Ok(part_files)
}
fn part_files_total_size(part_files: &[PathBuf]) -> io::Result<u64> {
part_files.iter().try_fold(0, |total, path| {
let metadata = fs::symlink_metadata(path)
.map_err(|error| io::Error::new(error.kind(), format!("failed to stat {}: {error}", path.display())))?;
if !metadata.file_type().is_file() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("expected regular part file at {}", path.display()),
));
}
total
.checked_add(metadata.len())
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "on-disk part size overflow"))
})
}
async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -123,8 +152,9 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
let content_length = head_response.content_length().unwrap_or(0);
assert_eq!(content_length as usize, original_size, "Content-Length should be original size");
let part_files = find_part_files(&env.temp_dir, COMPRESSION_TEST_BUCKET, object_key);
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
let part_files = find_part_files(&env.temp_dir, COMPRESSION_TEST_BUCKET, object_key)?;
assert!(!part_files.is_empty(), "expected on-disk part files for the compressed object");
let total_physical_size = part_files_total_size(&part_files)?;
assert!(
total_physical_size < original_size as u64,
@@ -246,9 +276,9 @@ async fn test_compression_multipart_roundtrip() -> Result<(), Box<dyn std::error
"Content-Length should be the logical object size"
);
let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key);
let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key)?;
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
let total_physical_size = part_files_total_size(&part_files)?;
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
@@ -366,9 +396,9 @@ async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(),
// This pattern compresses to roughly 1/50 of its logical size, so a comfortably loose 2x
// margin still proves the parts were stored compressed rather than raw or double-encoded.
let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key);
let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key)?;
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
let total_physical_size = part_files_total_size(&part_files)?;
assert!(
total_physical_size < (total_size as u64) / 2,
"Physical size {total_physical_size} should be far below the logical size {total_size} for high-ratio data"
@@ -522,9 +552,9 @@ async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), B
"Content-Length should be the logical object size"
);
let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key);
let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key)?;
assert!(!part_files.is_empty(), "expected on-disk part files for the copied object");
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
let total_physical_size = part_files_total_size(&part_files)?;
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (copied part compression applied)"
@@ -585,9 +615,9 @@ async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(),
"Content-Length should be the logical object size"
);
let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key);
let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key)?;
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
let total_physical_size = part_files_total_size(&part_files)?;
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
@@ -734,9 +764,9 @@ async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box<dyn std
"HEAD must report SSE-S3"
);
let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key);
let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key)?;
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
let total_physical_size = part_files_total_size(&part_files)?;
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (compress-then-encrypt applied)"