mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-02 02:08:41 +00:00
fix(s3): harden Snowball archive extraction (#6942)
* fix(s3): harden Snowball extract error boundaries * fix(s3): close Snowball extract compatibility gaps * fix(s3): verify Snowball request body completion * test(s3): reject forged Snowball streaming signatures * build(deps): pin Snowball archive parser limits * fix(s3): preserve Snowball trailer and member errors * docs(architecture): register Snowball tar fork cleanup * refactor(s3): route Snowball errors through object boundary * ci(deps): allow pinned tokio-tar source * ci(e2e): refresh Snowball smoke selection
This commit is contained in:
@@ -3456,6 +3456,62 @@ async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_ignore_dirs_skips_unauthorized_directory()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "signed-extract-ignore-dirs-auth";
|
||||
let archive_key = "bundle.tar";
|
||||
let allowed_member = "allowed/member.txt";
|
||||
let denied_directory = "denied/";
|
||||
let username = "snowball-ignore-dirs";
|
||||
let secret_key = "snowball-ignore-dirs-secret";
|
||||
let expected_body = b"allowed-body";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
create_restricted_user(&env, username, secret_key).await?;
|
||||
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [username] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{bucket}/{archive_key}"),
|
||||
format!("arn:aws:s3:::{bucket}/{allowed_member}")
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_client.put_bucket_policy().bucket(bucket).policy(policy).send().await?;
|
||||
|
||||
let restricted_client = restricted_user_client(&env, username, secret_key);
|
||||
let tar_bytes = make_tar(&[(allowed_member, expected_body)], &[denied_directory]).await;
|
||||
restricted_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
.body(ByteStream::from(tar_bytes))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
req.headers_mut().insert("x-amz-meta-snowball-ignore-dirs", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let stored = admin_client.get_object().bucket(bucket).key(allowed_member).send().await?;
|
||||
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_preserves_request_metadata_on_extracted_objects()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
|
||||
@@ -36,9 +36,10 @@
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER};
|
||||
use rustfs_signer::request_signature_v4::{SIGN_V4_ALGORITHM, get_scope, get_signature, get_signing_key};
|
||||
use std::fmt::Write as _;
|
||||
use std::io::Cursor;
|
||||
use time::macros::format_description;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use tracing::info;
|
||||
@@ -98,15 +99,37 @@ impl SigV4 {
|
||||
/// header AND folded into the canonical request — pass the hash of the
|
||||
/// body you *claim* to send, which may differ from what you actually send.
|
||||
fn sign(&self, method: &str, path: &str, canonical_query: &str, content_sha256: &str) -> SignedHeaders {
|
||||
let amz_date = amz_datetime(self.time);
|
||||
let signed_headers = "host;x-amz-content-sha256;x-amz-date";
|
||||
self.sign_with_extra_headers(method, path, canonical_query, content_sha256, &[])
|
||||
}
|
||||
|
||||
let canonical_headers = format!(
|
||||
"host:{host}\nx-amz-content-sha256:{sha}\nx-amz-date:{date}\n",
|
||||
host = self.host,
|
||||
sha = content_sha256,
|
||||
date = amz_date,
|
||||
);
|
||||
/// Sign additional request headers while preserving SigV4's lowercase,
|
||||
/// lexicographically sorted canonical-header representation.
|
||||
fn sign_with_extra_headers(
|
||||
&self,
|
||||
method: &str,
|
||||
path: &str,
|
||||
canonical_query: &str,
|
||||
content_sha256: &str,
|
||||
extra_signed_headers: &[(&str, &str)],
|
||||
) -> SignedHeaders {
|
||||
let amz_date = amz_datetime(self.time);
|
||||
let mut canonical_header_values = vec![
|
||||
("host", self.host.as_str()),
|
||||
("x-amz-content-sha256", content_sha256),
|
||||
("x-amz-date", amz_date.as_str()),
|
||||
];
|
||||
canonical_header_values.extend(extra_signed_headers.iter().copied());
|
||||
canonical_header_values.sort_unstable_by(|left, right| left.0.cmp(right.0));
|
||||
|
||||
let signed_headers = canonical_header_values
|
||||
.iter()
|
||||
.map(|(name, _)| *name)
|
||||
.collect::<Vec<_>>()
|
||||
.join(";");
|
||||
let mut canonical_headers = String::new();
|
||||
for (name, value) in canonical_header_values {
|
||||
let _ = writeln!(canonical_headers, "{name}:{value}");
|
||||
}
|
||||
let canonical_request =
|
||||
format!("{method}\n{path}\n{canonical_query}\n{canonical_headers}\n{signed_headers}\n{content_sha256}");
|
||||
|
||||
@@ -179,6 +202,34 @@ async fn setup(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn std::error
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn build_single_member_archive(
|
||||
member_key: &str,
|
||||
member_body: &[u8],
|
||||
) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(member_body.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder.append_data(&mut header, member_key, Cursor::new(member_body)).await?;
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
fn sha256_base64(data: &[u8]) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
base64_simd::STANDARD.encode_to_string(Sha256::digest(data))
|
||||
}
|
||||
|
||||
fn encode_unsigned_aws_chunked_with_sha256_trailer(decoded: &[u8]) -> Vec<u8> {
|
||||
let checksum = sha256_base64(decoded);
|
||||
let mut encoded = format!("{:x}\r\n", decoded.len()).into_bytes();
|
||||
encoded.extend_from_slice(decoded);
|
||||
encoded.extend_from_slice(b"\r\n0\r\n\r\n");
|
||||
encoded.extend_from_slice(format!("x-amz-checksum-sha256:{checksum}").as_bytes());
|
||||
encoded
|
||||
}
|
||||
|
||||
/// Positive control: a correctly hand-signed request must succeed. Without
|
||||
/// this, every negative assertion below could pass for the wrong reason (a
|
||||
/// broken signer that never produces a valid signature).
|
||||
@@ -249,6 +300,128 @@ async fn tampered_signature_returns_signature_does_not_match() -> Result<(), Box
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `STREAMING-UNSIGNED-PAYLOAD-TRAILER` disables per-chunk signatures, not the
|
||||
/// seed/header SigV4 signature. A forged request must be rejected before the
|
||||
/// Snowball handler can publish any archive member.
|
||||
#[tokio::test]
|
||||
async fn snowball_streaming_unsigned_trailer_rejects_forged_signature() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
setup(&mut env).await?;
|
||||
|
||||
let archive_key = "forged-streaming-snowball.tar";
|
||||
let member_key = "must-not-be-published.txt";
|
||||
let archive = build_single_member_archive(member_key, b"forged request payload").await?;
|
||||
let decoded_content_length = archive.len().to_string();
|
||||
let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(&archive);
|
||||
let path = format!("/{BUCKET}/{archive_key}");
|
||||
|
||||
let mut signer = SigV4::new(&env);
|
||||
signer.secret_key = "wrong-secret-for-forged-streaming-request".to_string();
|
||||
let extra_signed_headers = [
|
||||
("content-encoding", "aws-chunked"),
|
||||
("x-amz-decoded-content-length", decoded_content_length.as_str()),
|
||||
("x-amz-meta-snowball-auto-extract", "true"),
|
||||
("x-amz-trailer", "x-amz-checksum-sha256"),
|
||||
];
|
||||
let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
|
||||
|
||||
let response = local_http_client()
|
||||
.put(format!("{}{}", env.url, path))
|
||||
.header("authorization", &headers.authorization)
|
||||
.header("content-encoding", "aws-chunked")
|
||||
.header("x-amz-content-sha256", &headers.content_sha256)
|
||||
.header("x-amz-date", &headers.amz_date)
|
||||
.header("x-amz-decoded-content-length", &decoded_content_length)
|
||||
.header("x-amz-meta-snowball-auto-extract", "true")
|
||||
.header("x-amz-trailer", "x-amz-checksum-sha256")
|
||||
.body(encoded_body)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
assert_eq!(status.as_u16(), 403, "forged streaming signature must be 403, body:\n{body}");
|
||||
assert_error_code(&body, "SignatureDoesNotMatch");
|
||||
|
||||
let absent = env
|
||||
.create_s3_client()
|
||||
.get_object()
|
||||
.bucket(BUCKET)
|
||||
.key(member_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a forged streaming request must not publish a Snowball member");
|
||||
assert_eq!(absent.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Snowball must consume the complete aws-chunked body before reading the
|
||||
/// trailing checksum exported by s3s into the PutObject response.
|
||||
#[tokio::test]
|
||||
async fn snowball_streaming_unsigned_trailer_returns_sha256_checksum() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
setup(&mut env).await?;
|
||||
|
||||
let archive_key = "valid-streaming-snowball.tar";
|
||||
let member_key = "streaming-checksum-member.txt";
|
||||
let member_body = b"valid streaming Snowball payload";
|
||||
let archive = build_single_member_archive(member_key, member_body).await?;
|
||||
let expected_checksum = sha256_base64(&archive);
|
||||
let decoded_content_length = archive.len().to_string();
|
||||
let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(&archive);
|
||||
let path = format!("/{BUCKET}/{archive_key}");
|
||||
|
||||
let signer = SigV4::new(&env);
|
||||
let extra_signed_headers = [
|
||||
("content-encoding", "aws-chunked"),
|
||||
("x-amz-decoded-content-length", decoded_content_length.as_str()),
|
||||
("x-amz-meta-snowball-auto-extract", "true"),
|
||||
("x-amz-sdk-checksum-algorithm", "SHA256"),
|
||||
("x-amz-trailer", "x-amz-checksum-sha256"),
|
||||
];
|
||||
let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
|
||||
|
||||
let response = local_http_client()
|
||||
.put(format!("{}{}", env.url, path))
|
||||
.header("authorization", &headers.authorization)
|
||||
.header("content-encoding", "aws-chunked")
|
||||
.header("x-amz-content-sha256", &headers.content_sha256)
|
||||
.header("x-amz-date", &headers.amz_date)
|
||||
.header("x-amz-decoded-content-length", &decoded_content_length)
|
||||
.header("x-amz-meta-snowball-auto-extract", "true")
|
||||
.header("x-amz-sdk-checksum-algorithm", "SHA256")
|
||||
.header("x-amz-trailer", "x-amz-checksum-sha256")
|
||||
.body(encoded_body)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let response_checksum = response
|
||||
.headers()
|
||||
.get("x-amz-checksum-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned);
|
||||
let response_body = response.text().await?;
|
||||
assert_eq!(status.as_u16(), 200, "valid streaming Snowball PUT failed, body:\n{response_body}");
|
||||
assert_eq!(response_checksum.as_deref(), Some(expected_checksum.as_str()));
|
||||
|
||||
let member = env
|
||||
.create_s3_client()
|
||||
.get_object()
|
||||
.bucket(BUCKET)
|
||||
.key(member_key)
|
||||
.send()
|
||||
.await?;
|
||||
let stored = member.body.collect().await?.into_bytes();
|
||||
assert_eq!(stored.as_ref(), member_body);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// (b) A valid AccessKeyId paired with the wrong secret key must be rejected
|
||||
/// with SignatureDoesNotMatch / 403.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -17,8 +17,9 @@ mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
use std::io::{Cursor, Write};
|
||||
|
||||
async fn build_test_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
@@ -69,12 +70,50 @@ mod tests {
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
|
||||
let path = format!("../{victim_bucket}/evil-injected.txt");
|
||||
let data = b"injected-body";
|
||||
async fn build_archive_with_invalid_checksum() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut archive = build_test_archive().await?;
|
||||
archive[0] ^= 1;
|
||||
Ok(archive)
|
||||
}
|
||||
|
||||
async fn build_archive_with_negative_gnu_mtime() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(b"negative-mtime-body".len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.as_old_mut().mtime.fill(0xff);
|
||||
builder
|
||||
.append_data(&mut header, "negative-mtime.txt", Cursor::new(b"negative-mtime-body".as_slice()))
|
||||
.await?;
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
fn gzip_member(payload: &[u8]) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(payload)?;
|
||||
Ok(encoder.finish()?)
|
||||
}
|
||||
|
||||
async fn build_concatenated_gzip_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let archive = build_test_archive().await?;
|
||||
let split_at = archive.len() / 2;
|
||||
let mut encoded = gzip_member(&archive[..split_at])?;
|
||||
encoded.extend(gzip_member(&archive[split_at..])?);
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
async fn build_gzip_archive_with_invalid_crc() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut encoded = gzip_member(&build_test_archive().await?)?;
|
||||
let crc_offset = encoded.len().checked_sub(8).expect("gzip fixture must contain a trailer");
|
||||
encoded[crc_offset] ^= 1;
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
fn append_raw_tar_entry_with_type(archive: &mut Vec<u8>, path: &[u8], data: &[u8], entry_type: u8) {
|
||||
assert!(path.len() <= 100, "raw TAR fixture path must fit in the name field");
|
||||
let mut header = [0u8; 512];
|
||||
|
||||
header[..path.len()].copy_from_slice(path.as_bytes());
|
||||
header[..path.len()].copy_from_slice(path);
|
||||
header[100..108].copy_from_slice(b"0000644\0");
|
||||
header[108..116].copy_from_slice(b"0000000\0");
|
||||
header[116..124].copy_from_slice(b"0000000\0");
|
||||
@@ -82,7 +121,7 @@ mod tests {
|
||||
header[124..136].copy_from_slice(size.as_bytes());
|
||||
header[136..148].copy_from_slice(b"00000000000\0");
|
||||
header[148..156].fill(b' ');
|
||||
header[156] = b'0';
|
||||
header[156] = entry_type;
|
||||
header[257..263].copy_from_slice(b"ustar\0");
|
||||
header[263..265].copy_from_slice(b"00");
|
||||
|
||||
@@ -90,11 +129,36 @@ mod tests {
|
||||
let checksum = format!("{:06o}\0 ", checksum);
|
||||
header[148..156].copy_from_slice(checksum.as_bytes());
|
||||
|
||||
let mut archive = Vec::new();
|
||||
archive.extend_from_slice(&header);
|
||||
archive.extend_from_slice(data);
|
||||
let padding = (512 - (data.len() % 512)) % 512;
|
||||
archive.extend(std::iter::repeat_n(0, padding));
|
||||
}
|
||||
|
||||
fn append_raw_tar_entry(archive: &mut Vec<u8>, path: &[u8], data: &[u8]) {
|
||||
append_raw_tar_entry_with_type(archive, path, data, b'0');
|
||||
}
|
||||
|
||||
fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
|
||||
let path = format!("../{victim_bucket}/evil-injected.txt");
|
||||
let mut archive = Vec::new();
|
||||
append_raw_tar_entry(&mut archive, path.as_bytes(), b"injected-body");
|
||||
archive.extend_from_slice(&[0u8; 1024]);
|
||||
archive
|
||||
}
|
||||
|
||||
fn build_archive_with_invalid_utf8_entry() -> Vec<u8> {
|
||||
let mut archive = Vec::new();
|
||||
append_raw_tar_entry(&mut archive, b"invalid-\xff.txt", b"ignored-body");
|
||||
append_raw_tar_entry(&mut archive, b"valid.txt", b"valid-body");
|
||||
archive.extend_from_slice(&[0u8; 1024]);
|
||||
archive
|
||||
}
|
||||
|
||||
fn build_archive_with_invalid_utf8_symlink() -> Vec<u8> {
|
||||
let mut archive = Vec::new();
|
||||
append_raw_tar_entry_with_type(&mut archive, b"invalid-\xff-link", b"", b'2');
|
||||
append_raw_tar_entry(&mut archive, b"valid.txt", b"valid-body");
|
||||
archive.extend_from_slice(&[0u8; 1024]);
|
||||
archive
|
||||
}
|
||||
@@ -263,6 +327,113 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_accepts_negative_gnu_mtime() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-negative-mtime";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_archive_with_negative_gnu_mtime().await?))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let object = client.get_object().bucket(bucket).key("negative-mtime.txt").send().await?;
|
||||
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"negative-mtime-body");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_consumes_concatenated_gzip_members() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-concatenated-gzip";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar.gz")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_concatenated_gzip_archive().await?))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let object = client.get_object().bucket(bucket).key("root.txt").send().await?;
|
||||
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"root payload\n");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_gzip_crc_error_when_ignore_errors_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-gzip-crc-ignore-errors";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar.gz")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(build_gzip_archive_with_invalid_crc().await?))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("gzip integrity failures must remain fatal under ignore-errors");
|
||||
|
||||
assert_eq!(err.into_service_error().code(), Some("InvalidArgument"));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_mismatched_content_md5() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-content-md5";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.content_md5("AAAAAAAAAAAAAAAAAAAAAA==")
|
||||
.body(ByteStream::from(build_test_archive().await?))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("mismatched Content-MD5 must fail after the raw body reaches EOF");
|
||||
|
||||
assert_eq!(err.into_service_error().code(), Some("BadDigest"));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_ignores_invalid_entries_when_requested() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -299,7 +470,100 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_parent_dir_entry_without_cross_bucket_write()
|
||||
async fn snowball_auto_extract_skips_non_utf8_symlink_without_ignore_errors() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-invalid-utf8-link";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_archive_with_invalid_utf8_symlink()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let valid = client.get_object().bucket(bucket).key("valid.txt").send().await?;
|
||||
assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
|
||||
let listed = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
|
||||
assert_eq!(keys, vec!["valid.txt"]);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_skips_non_utf8_member_without_lossy_key_collision() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-invalid-utf8";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(build_archive_with_invalid_utf8_entry()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let valid = client.get_object().bucket(bucket).key("valid.txt").send().await?;
|
||||
assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
|
||||
let listed = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
|
||||
assert_eq!(keys, vec!["valid.txt"]);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_corrupt_tar_when_ignore_errors_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-corrupt-ignore-errors";
|
||||
let archive = build_archive_with_invalid_checksum().await?;
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(archive))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("corrupt TAR structure must remain fatal under ignore-errors");
|
||||
assert_eq!(err.into_service_error().code(), Some("InvalidArgument"));
|
||||
|
||||
let listed = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
assert!(listed.contents().is_empty(), "corrupt archive must not produce objects");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_parent_dir_entry_even_when_ignore_errors_enabled()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -319,6 +583,7 @@ mod tests {
|
||||
.bucket(attacker_bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(archive))
|
||||
.send()
|
||||
.await
|
||||
|
||||
Reference in New Issue
Block a user