mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 08:27:06 +00:00
refactor: move ecstore owner layout modules (#3932)
This commit is contained in:
Zhengchao An
@@ -0,0 +1,339 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::{self, DiskAPI as _, DiskStore, error::DiskError};
|
||||
use crate::erasure_coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
|
||||
use bytes::Bytes;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use std::io::Cursor;
|
||||
use std::time::Instant;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::debug;
|
||||
|
||||
/// Create a BitrotReader from either inline data or disk file stream
|
||||
///
|
||||
/// # Parameters
|
||||
/// * `inline_data` - Optional inline data, if present, will use Cursor to read from memory
|
||||
/// * `disk` - Optional disk reference for file stream reading
|
||||
/// * `bucket` - Bucket name for file path
|
||||
/// * `path` - File path within the bucket
|
||||
/// * `offset` - Starting offset for reading
|
||||
/// * `length` - Length to read
|
||||
/// * `shard_size` - Shard size for erasure coding
|
||||
/// * `checksum_algo` - Hash algorithm for bitrot verification
|
||||
/// * `skip_verify` - If true, skip checksum verification
|
||||
/// * `use_zero_copy` - If true, use zero-copy read (mmap on Unix)
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn create_bitrot_reader(
|
||||
inline_data: Option<&[u8]>,
|
||||
disk: Option<&DiskStore>,
|
||||
bucket: &str,
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
skip_verify: bool,
|
||||
use_zero_copy: bool,
|
||||
) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
|
||||
// Calculate the total length to read, including the checksum overhead
|
||||
let length = length.div_ceil(shard_size) * checksum_algo.size() + length;
|
||||
let offset = offset.div_ceil(shard_size) * checksum_algo.size() + offset;
|
||||
if let Some(data) = inline_data {
|
||||
// Use inline data
|
||||
let mut rd = Cursor::new(Bytes::copy_from_slice(data));
|
||||
// Apply the computed offset so inline data matches disk read behavior
|
||||
rd.set_position(offset as u64);
|
||||
let reader = BitrotReader::new(
|
||||
Box::new(rd) as Box<dyn AsyncRead + Send + Sync + Unpin>,
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
skip_verify,
|
||||
);
|
||||
Ok(Some(reader))
|
||||
} else if let Some(disk) = disk {
|
||||
// Read from disk
|
||||
if use_zero_copy && disk.is_local() {
|
||||
// Try zero-copy read first (uses mmap on Unix)
|
||||
let start = Instant::now();
|
||||
match disk.read_file_zero_copy(bucket, path, offset, length).await {
|
||||
Ok(bytes) => {
|
||||
let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
// Record zero-copy metrics
|
||||
rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms);
|
||||
|
||||
// Log successful zero-copy read
|
||||
debug!(
|
||||
size = bytes.len(),
|
||||
path = %path,
|
||||
"zero_copy_read_success"
|
||||
);
|
||||
|
||||
// Wrap Bytes in Cursor for AsyncRead
|
||||
// The Bytes is reference-counted, so this is zero-copy
|
||||
let rd = Cursor::new(bytes);
|
||||
let reader = BitrotReader::new(
|
||||
Box::new(rd) as Box<dyn AsyncRead + Send + Sync + Unpin>,
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
skip_verify,
|
||||
);
|
||||
Ok(Some(reader))
|
||||
}
|
||||
Err(e) => {
|
||||
// Record zero-copy fallback
|
||||
rustfs_io_metrics::record_zero_copy_fallback(&format!("{:?}", e));
|
||||
|
||||
// Log zero-copy fallback
|
||||
debug!(
|
||||
reason = %format!("{:?}", e),
|
||||
path = %path,
|
||||
"zero_copy_fallback"
|
||||
);
|
||||
|
||||
// Fall back to regular stream read on error
|
||||
match disk.read_file_stream(bucket, path, offset, length).await {
|
||||
Ok(rd) => {
|
||||
let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
|
||||
Ok(Some(reader))
|
||||
}
|
||||
Err(_e2) => {
|
||||
// Return the original error from zero-copy attempt
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Use regular stream read
|
||||
match disk.read_file_stream(bucket, path, offset, length).await {
|
||||
Ok(rd) => {
|
||||
let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
|
||||
Ok(Some(reader))
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Neither inline data nor disk available
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new BitrotWriterWrapper based on the provided parameters
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `is_inline_buffer`: If true, creates an in-memory buffer writer; if false, uses disk storage
|
||||
/// - `disk`: Optional disk instance for file creation (used when is_inline_buffer is false)
|
||||
/// - `shard_size`: Size of each shard for bitrot calculation
|
||||
/// - `checksum_algo`: Hash algorithm to use for bitrot verification
|
||||
/// - `volume`: Volume/bucket name for disk storage
|
||||
/// - `path`: File path for disk storage
|
||||
/// - `length`: Expected file length for disk storage
|
||||
///
|
||||
/// # Returns
|
||||
/// A Result containing the BitrotWriterWrapper or an error
|
||||
pub async fn create_bitrot_writer(
|
||||
is_inline_buffer: bool,
|
||||
disk: Option<&DiskStore>,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
length: i64,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
) -> disk::error::Result<BitrotWriterWrapper> {
|
||||
let writer = if is_inline_buffer {
|
||||
CustomWriter::new_inline_buffer()
|
||||
} else if let Some(disk) = disk {
|
||||
let length = if length > 0 {
|
||||
let length = length as usize;
|
||||
(length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let file = disk.create_file("", volume, path, length).await?;
|
||||
CustomWriter::new_tokio_writer(file)
|
||||
} else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
|
||||
Ok(BitrotWriterWrapper::new(writer, shard_size, checksum_algo))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_reader_with_inline_data() {
|
||||
let test_data = b"hello world test data";
|
||||
let shard_size = 16;
|
||||
let checksum_algo = HashAlgorithm::HighwayHash256S;
|
||||
|
||||
let result = create_bitrot_reader(
|
||||
Some(test_data),
|
||||
None,
|
||||
"test-bucket",
|
||||
"test-path",
|
||||
0,
|
||||
0,
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
assert!(result.unwrap().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_reader_with_zero_copy_enabled() {
|
||||
let test_data = b"hello world test data";
|
||||
let shard_size = 16;
|
||||
let checksum_algo = HashAlgorithm::HighwayHash256S;
|
||||
|
||||
// Test with zero-copy enabled (should work the same for inline data)
|
||||
let result = create_bitrot_reader(
|
||||
Some(test_data),
|
||||
None,
|
||||
"test-bucket",
|
||||
"test-path",
|
||||
0,
|
||||
0,
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
assert!(result.unwrap().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_reader_with_inline_offset_starts_at_requested_shard() {
|
||||
let shard_size = 4;
|
||||
let checksum_algo = HashAlgorithm::HighwayHash256S;
|
||||
let payload = b"abcdefghijkl";
|
||||
|
||||
let mut writer = create_bitrot_writer(
|
||||
true,
|
||||
None,
|
||||
"test-volume",
|
||||
"test-path",
|
||||
payload.len() as i64,
|
||||
shard_size,
|
||||
checksum_algo.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("inline bitrot writer");
|
||||
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer.write(chunk).await.expect("write chunk");
|
||||
}
|
||||
|
||||
let inline_data = writer.into_inline_data().expect("inline buffer");
|
||||
let mut reader = create_bitrot_reader(
|
||||
Some(&inline_data),
|
||||
None,
|
||||
"test-bucket",
|
||||
"test-path",
|
||||
shard_size,
|
||||
shard_size,
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("create reader")
|
||||
.expect("reader");
|
||||
|
||||
let mut out = [0u8; 4];
|
||||
let n = reader.read(&mut out).await.expect("read second shard");
|
||||
|
||||
assert_eq!(n, shard_size);
|
||||
assert_eq!(&out[..n], b"efgh");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_reader_without_data_or_disk() {
|
||||
let shard_size = 16;
|
||||
let checksum_algo = HashAlgorithm::HighwayHash256S;
|
||||
|
||||
let result =
|
||||
create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo, false, false).await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
assert!(result.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_writer_inline() {
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
|
||||
let wrapper = create_bitrot_writer(
|
||||
true, // is_inline_buffer
|
||||
None, // disk not needed for inline buffer
|
||||
"test-volume",
|
||||
"test-path",
|
||||
1024, // length
|
||||
1024, // shard_size
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(wrapper.is_ok());
|
||||
let mut wrapper = wrapper.unwrap();
|
||||
|
||||
// Test writing some data
|
||||
let test_data = b"hello world";
|
||||
let result = wrapper.write(test_data).await;
|
||||
assert!(result.is_ok());
|
||||
|
||||
// Test getting inline data
|
||||
let inline_data = wrapper.into_inline_data();
|
||||
assert!(inline_data.is_some());
|
||||
// The inline data should contain both hash and data
|
||||
let data = inline_data.unwrap();
|
||||
assert!(!data.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_writer_disk_without_disk() {
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
|
||||
// Test error case: trying to create disk writer without providing disk instance
|
||||
let wrapper = create_bitrot_writer(
|
||||
false, // is_inline_buffer = false, so needs disk
|
||||
None, // disk = None, should cause error
|
||||
"test-volume",
|
||||
"test-path",
|
||||
1024, // length
|
||||
1024, // shard_size
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(wrapper.is_err());
|
||||
let error = wrapper.unwrap_err();
|
||||
println!("error: {error:?}");
|
||||
assert_eq!(error, DiskError::DiskNotFound);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_utils::string::{has_pattern, has_string_suffix_in_slice, match_simple};
|
||||
use std::env;
|
||||
use std::sync::OnceLock;
|
||||
use tracing::debug;
|
||||
|
||||
pub const MIN_DISK_COMPRESSIBLE_SIZE: usize = 4096;
|
||||
|
||||
// Environment variable name to control whether object disk compression is enabled.
|
||||
pub const ENV_DISK_COMPRESSION_ENABLED: &str = "RUSTFS_COMPRESSION_ENABLED";
|
||||
|
||||
// Environment variable for file extensions to include in object disk compression.
|
||||
pub const ENV_DISK_COMPRESSION_EXTENSIONS: &str = "RUSTFS_COMPRESSION_EXTENSIONS";
|
||||
|
||||
// Environment variable for MIME types to include in object disk compression.
|
||||
pub const ENV_DISK_COMPRESSION_MIME_TYPES: &str = "RUSTFS_COMPRESSION_MIME_TYPES";
|
||||
|
||||
// Environment variable for additional extensions to exclude from compression (comma-separated, e.g. ".foo,.bar")
|
||||
pub const ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS: &str = "RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS";
|
||||
|
||||
pub const DEFAULT_DISK_COMPRESS_EXTENSIONS: &str = ".txt,.log,.csv,.json,.tar,.xml,.bin";
|
||||
pub const DEFAULT_DISK_COMPRESS_MIME_TYPES: &str = "text/*,application/json,application/xml,binary/octet-stream";
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DiskCompressionConfig {
|
||||
enabled: bool,
|
||||
extensions: Vec<String>,
|
||||
mime_types: Vec<String>,
|
||||
added_exclude_extensions: Vec<String>,
|
||||
}
|
||||
|
||||
/// Parses RUSTFS_COMPRESSION_ENABLED. Called once at first use via OnceLock.
|
||||
fn parse_disk_compression_enabled() -> bool {
|
||||
env::var(ENV_DISK_COMPRESSION_ENABLED)
|
||||
.map(|s| matches!(s.to_ascii_lowercase().as_str(), "true" | "on" | "1"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn parse_csv(value: &str) -> Vec<String> {
|
||||
value
|
||||
.split(',')
|
||||
.map(|v| v.trim().to_ascii_lowercase())
|
||||
.filter(|v| !v.is_empty())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn normalize_extensions(extensions: Vec<String>) -> Vec<String> {
|
||||
extensions
|
||||
.into_iter()
|
||||
.map(|v| if v.starts_with('.') { v } else { format!(".{v}") })
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn has_config_pattern(patterns: &[String], match_str: &str) -> bool {
|
||||
patterns.iter().any(|pattern| match_simple(pattern, match_str))
|
||||
}
|
||||
|
||||
fn has_object_content_encoding(headers: &http::HeaderMap) -> bool {
|
||||
for value in headers.get_all(http::header::CONTENT_ENCODING) {
|
||||
let Ok(content_encoding) = value.to_str() else {
|
||||
return true;
|
||||
};
|
||||
|
||||
for encoding in content_encoding.split(',').map(str::trim) {
|
||||
if !encoding.is_empty() && !encoding.eq_ignore_ascii_case("identity") && !encoding.eq_ignore_ascii_case("aws-chunked")
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn parse_disk_compression_extensions() -> Vec<String> {
|
||||
let extensions = env::var(ENV_DISK_COMPRESSION_EXTENSIONS).unwrap_or_else(|_| DEFAULT_DISK_COMPRESS_EXTENSIONS.to_string());
|
||||
normalize_extensions(parse_csv(&extensions))
|
||||
}
|
||||
|
||||
fn parse_disk_compression_mime_types() -> Vec<String> {
|
||||
let mime_types = env::var(ENV_DISK_COMPRESSION_MIME_TYPES).unwrap_or_else(|_| DEFAULT_DISK_COMPRESS_MIME_TYPES.to_string());
|
||||
parse_csv(&mime_types)
|
||||
}
|
||||
|
||||
/// Parses RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS (comma-separated). Called once at first use via OnceLock.
|
||||
pub(crate) fn parse_added_exclude_extensions() -> Vec<String> {
|
||||
env::var(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS)
|
||||
.ok()
|
||||
.map(|s| normalize_extensions(parse_csv(&s)))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn parse_disk_compression_config() -> DiskCompressionConfig {
|
||||
DiskCompressionConfig {
|
||||
enabled: parse_disk_compression_enabled(),
|
||||
extensions: parse_disk_compression_extensions(),
|
||||
mime_types: parse_disk_compression_mime_types(),
|
||||
added_exclude_extensions: parse_added_exclude_extensions(),
|
||||
}
|
||||
}
|
||||
|
||||
// Parsed once at first use, then reused for all disk compression checks.
|
||||
static DISK_COMPRESSION_CONFIG: OnceLock<DiskCompressionConfig> = OnceLock::new();
|
||||
|
||||
// Some standard object extensions which we strictly dis-allow for compression.
|
||||
#[rustfmt::skip]
|
||||
pub const DISK_COMPRESSION_EXCLUDED_EXTENSIONS: &[&str] = &[
|
||||
// Compressed archives
|
||||
".gz", ".bz2", ".rar", ".zip", ".7z", ".xz", ".zst", ".lz4", ".br", ".lzo", ".sz", ".tgz",
|
||||
// Images
|
||||
".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".heic", ".heif", ".jxl",
|
||||
// Video
|
||||
".mp4", ".mkv", ".mov", ".avi", ".wmv", ".flv", ".webm", ".m4v", ".mpeg", ".mpg",
|
||||
// Audio
|
||||
".mp3", ".aac", ".ogg", ".flac", ".wma", ".m4a", ".opus",
|
||||
// Documents (internally compressed)
|
||||
".pdf", ".docx", ".xlsx", ".pptx",
|
||||
// Package formats
|
||||
".deb", ".rpm", ".jar", ".war", ".apk",
|
||||
// Web fonts
|
||||
".woff", ".woff2",
|
||||
];
|
||||
|
||||
// Some standard content-types which we strictly dis-allow for compression.
|
||||
pub const DISK_COMPRESSION_EXCLUDED_CONTENT_TYPES: &[&str] = &[
|
||||
"video/*",
|
||||
"audio/*",
|
||||
"image/*",
|
||||
// Archive formats (compressed)
|
||||
"application/zip",
|
||||
"application/gzip",
|
||||
"application/x-gzip",
|
||||
"application/x-zip-compressed",
|
||||
"application/x-compress",
|
||||
"application/x-spoon",
|
||||
"application/x-rar-compressed",
|
||||
"application/x-7z-compressed",
|
||||
"application/x-bzip",
|
||||
"application/x-bzip2",
|
||||
"application/x-xz",
|
||||
"application/x-lzip",
|
||||
"application/x-lzma",
|
||||
"application/x-lzop",
|
||||
"application/zstd",
|
||||
"application/x-zstd",
|
||||
// Archive formats (uncompressed containers that are typically not further compressible)
|
||||
"application/x-tar",
|
||||
"application/tar",
|
||||
"application/pdf",
|
||||
"application/wasm",
|
||||
"font/*",
|
||||
];
|
||||
|
||||
pub fn is_disk_compressible(headers: &http::HeaderMap, object_name: &str) -> bool {
|
||||
is_disk_compressible_with_config(headers, object_name, DISK_COMPRESSION_CONFIG.get_or_init(parse_disk_compression_config))
|
||||
}
|
||||
|
||||
fn is_disk_compressible_with_config(headers: &http::HeaderMap, object_name: &str, config: &DiskCompressionConfig) -> bool {
|
||||
// Check if disk compression is enabled (read once at first use, then fixed for process lifetime)
|
||||
if !config.enabled {
|
||||
debug!("Disk compression is disabled by environment variable");
|
||||
return false;
|
||||
}
|
||||
|
||||
let content_type = headers.get("content-type").and_then(|s| s.to_str().ok()).unwrap_or("");
|
||||
|
||||
if has_object_content_encoding(headers) {
|
||||
debug!("object_name: {} is already content-encoded; skipping disk compression", object_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if has_string_suffix_in_slice(object_name, DISK_COMPRESSION_EXCLUDED_EXTENSIONS) {
|
||||
debug!("object_name: {} is not disk-compressible", object_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if !config.added_exclude_extensions.is_empty() && has_string_suffix_in_slice(object_name, &config.added_exclude_extensions) {
|
||||
debug!("object_name: {} is not disk-compressible (added exclusion)", object_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if !content_type.is_empty() && has_pattern(DISK_COMPRESSION_EXCLUDED_CONTENT_TYPES, content_type) {
|
||||
debug!("content_type: {} is not disk-compressible", content_type);
|
||||
return false;
|
||||
}
|
||||
|
||||
if config.extensions.is_empty() && config.mime_types.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
if !config.extensions.is_empty() && has_string_suffix_in_slice(object_name, &config.extensions) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if !content_type.is_empty() && !config.mime_types.is_empty() && has_config_pattern(&config.mime_types, content_type) {
|
||||
return true;
|
||||
}
|
||||
|
||||
debug!("object_name: {} does not match disk compression include filters", object_name);
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_disk_compression_enabled() {
|
||||
temp_env::with_var(ENV_DISK_COMPRESSION_ENABLED, Some("true"), || {
|
||||
assert!(parse_disk_compression_enabled());
|
||||
});
|
||||
temp_env::with_var(ENV_DISK_COMPRESSION_ENABLED, Some("on"), || {
|
||||
assert!(parse_disk_compression_enabled());
|
||||
});
|
||||
temp_env::with_var(ENV_DISK_COMPRESSION_ENABLED, Some("false"), || {
|
||||
assert!(!parse_disk_compression_enabled());
|
||||
});
|
||||
temp_env::with_var(ENV_DISK_COMPRESSION_ENABLED, Some("FALSE"), || {
|
||||
assert!(!parse_disk_compression_enabled());
|
||||
});
|
||||
temp_env::with_var_unset(ENV_DISK_COMPRESSION_ENABLED, || {
|
||||
assert!(!parse_disk_compression_enabled());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_disk_compression_includes() {
|
||||
temp_env::with_var(ENV_DISK_COMPRESSION_EXTENSIONS, Some(".txt,log, .json"), || {
|
||||
assert_eq!(parse_disk_compression_extensions(), [".txt", ".log", ".json"]);
|
||||
});
|
||||
temp_env::with_var(ENV_DISK_COMPRESSION_MIME_TYPES, Some("text/*, application/json"), || {
|
||||
assert_eq!(parse_disk_compression_mime_types(), ["text/*", "application/json"]);
|
||||
});
|
||||
temp_env::with_var_unset(ENV_DISK_COMPRESSION_EXTENSIONS, || {
|
||||
assert_eq!(
|
||||
parse_disk_compression_extensions(),
|
||||
[".txt", ".log", ".csv", ".json", ".tar", ".xml", ".bin"]
|
||||
);
|
||||
});
|
||||
temp_env::with_var_unset(ENV_DISK_COMPRESSION_MIME_TYPES, || {
|
||||
assert_eq!(
|
||||
parse_disk_compression_mime_types(),
|
||||
["text/*", "application/json", "application/xml", "binary/octet-stream"]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
fn test_config() -> DiskCompressionConfig {
|
||||
DiskCompressionConfig {
|
||||
enabled: true,
|
||||
extensions: normalize_extensions(parse_csv(DEFAULT_DISK_COMPRESS_EXTENSIONS)),
|
||||
mime_types: parse_csv(DEFAULT_DISK_COMPRESS_MIME_TYPES),
|
||||
added_exclude_extensions: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_disk_compressible() {
|
||||
use http::HeaderMap;
|
||||
|
||||
let config = test_config();
|
||||
let mut headers = HeaderMap::new();
|
||||
// Test non-compressible extensions - compressed archives
|
||||
headers.insert("content-type", "text/plain".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.gz", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.zip", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.7z", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.zst", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.br", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.tgz", &config));
|
||||
|
||||
// Test non-compressible extensions - images
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.jpg", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.jpeg", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.png", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.gif", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.webp", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.avif", &config));
|
||||
|
||||
// Test non-compressible extensions - video
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.mp4", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.mkv", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.webm", &config));
|
||||
|
||||
// Test non-compressible extensions - audio
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.mp3", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.aac", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.ogg", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.flac", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.opus", &config));
|
||||
|
||||
// Test non-compressible extensions - documents
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.pdf", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.docx", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.xlsx", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.pptx", &config));
|
||||
|
||||
// Test non-compressible extensions - packages
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.deb", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.rpm", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.jar", &config));
|
||||
|
||||
// Test non-compressible extensions - web fonts
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.woff2", &config));
|
||||
|
||||
// Test non-compressible content types
|
||||
headers.insert("content-type", "video/mp4".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert("content-type", "audio/mpeg".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert("content-type", "image/png".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert("content-type", "image/webp".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert("content-type", "application/zip".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert("content-type", "application/x-gzip".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert("content-type", "application/x-7z-compressed".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert("content-type", "application/pdf".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert("content-type", "font/woff2".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
// Test compressible cases
|
||||
headers.insert("content-type", "text/plain".parse().expect("valid content type"));
|
||||
assert!(is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
assert!(is_disk_compressible_with_config(&headers, "file.log", &config));
|
||||
|
||||
headers.insert("content-type", "text/html".parse().expect("valid content type"));
|
||||
assert!(is_disk_compressible_with_config(&headers, "file.html", &config));
|
||||
|
||||
headers.insert("content-type", "application/json".parse().expect("valid content type"));
|
||||
assert!(is_disk_compressible_with_config(&headers, "file.json", &config));
|
||||
|
||||
headers.insert("content-type", "application/octet-stream".parse().expect("valid content type"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.data", &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_content_encoding_skips_disk_compression() {
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
|
||||
let config = test_config();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("content-type", "text/plain".parse().expect("valid content type"));
|
||||
|
||||
headers.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("zstd"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("gzip"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("aws-chunked,gzip"));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("aws-chunked"));
|
||||
assert!(is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
|
||||
headers.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("identity"));
|
||||
assert!(is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_added_exclude_compress_extensions_parsing() {
|
||||
temp_env::with_var(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, Some(".foo,.bar"), || {
|
||||
let added = parse_added_exclude_extensions();
|
||||
assert_eq!(added, [".foo", ".bar"]);
|
||||
});
|
||||
temp_env::with_var(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, Some("baz, .qux"), || {
|
||||
let added = parse_added_exclude_extensions();
|
||||
assert_eq!(added, [".baz", ".qux"]);
|
||||
});
|
||||
temp_env::with_var_unset(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, || {
|
||||
let added = parse_added_exclude_extensions();
|
||||
assert!(added.is_empty());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_added_exclude_compress_extensions_excludes_included_extension() {
|
||||
use http::HeaderMap;
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("content-type", "text/plain".parse().expect("valid content type"));
|
||||
let mut config = test_config();
|
||||
config.added_exclude_extensions = vec![".txt".to_string()];
|
||||
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config));
|
||||
assert!(is_disk_compressible_with_config(&headers, "file.log", &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_includes_compress_everything_except_exclusions() {
|
||||
use http::HeaderMap;
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("content-type", "application/octet-stream".parse().expect("valid content type"));
|
||||
let config = DiskCompressionConfig {
|
||||
enabled: true,
|
||||
extensions: Vec::new(),
|
||||
mime_types: Vec::new(),
|
||||
added_exclude_extensions: Vec::new(),
|
||||
};
|
||||
|
||||
assert!(is_disk_compressible_with_config(&headers, "file.data", &config));
|
||||
assert!(!is_disk_compressible_with_config(&headers, "file.zip", &config));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,767 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub use rustfs_rio_v2::*;
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub use rustfs_rio::*;
|
||||
|
||||
use bytes::Bytes;
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
use std::str::FromStr;
|
||||
use tokio::io::AsyncRead;
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
const MINIO_S2_COMPRESSION_SCHEME: &str = "klauspost/compress/s2";
|
||||
#[cfg(feature = "rio-v2")]
|
||||
const ENCRYPTED_S2_PADDING_MULTIPLE: usize = 256;
|
||||
|
||||
pub const fn backend_name() -> &'static str {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
"rio-v2"
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
{
|
||||
"legacy-rio"
|
||||
}
|
||||
}
|
||||
|
||||
pub fn compression_metadata_value(algorithm: CompressionAlgorithm) -> String {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
let _ = algorithm;
|
||||
MINIO_S2_COMPRESSION_SCHEME.to_string()
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
{
|
||||
algorithm.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn compression_scheme_to_algorithm(scheme: &str) -> std::io::Result<CompressionAlgorithm> {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) {
|
||||
// rio_v2 currently routes all compressed-object handling through the S2
|
||||
// reader implementation, so the enum is only a placeholder token here.
|
||||
return Ok(CompressionAlgorithm::default());
|
||||
}
|
||||
|
||||
CompressionAlgorithm::from_str(scheme)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReadCompressionBackend {
|
||||
Legacy,
|
||||
V2,
|
||||
}
|
||||
|
||||
pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(CompressionAlgorithm, ReadCompressionBackend)> {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) {
|
||||
return Ok((CompressionAlgorithm::default(), ReadCompressionBackend::V2));
|
||||
}
|
||||
|
||||
Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReadEncryptionBackend {
|
||||
Legacy,
|
||||
V2,
|
||||
}
|
||||
|
||||
pub fn compression_index_storage_bytes(index: &Index) -> Bytes {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
minio_index_storage_bytes(index)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
{
|
||||
index.clone().into_vec()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode_compression_index_bytes(bytes: &Bytes) -> Option<Index> {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
if let Some(decoded) = decode_minio_index_bytes(bytes) {
|
||||
return Some(decoded);
|
||||
}
|
||||
}
|
||||
|
||||
let mut decoded = Index::new();
|
||||
if decoded.load(bytes.as_ref()).is_ok() {
|
||||
return Some(decoded);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
let restored = restore_legacy_index_headers(bytes.as_ref());
|
||||
let mut decoded = Index::new();
|
||||
if decoded.load(&restored).is_ok() {
|
||||
return Some(decoded);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub fn compression_reader<R>(reader: R, algorithm: CompressionAlgorithm, encrypted: bool) -> CompressReader<R>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync,
|
||||
{
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
if encrypted {
|
||||
return CompressReader::with_encrypted_padding(reader, algorithm);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let _ = encrypted;
|
||||
|
||||
CompressReader::new(reader, algorithm)
|
||||
}
|
||||
|
||||
pub fn decompression_reader<R>(
|
||||
reader: R,
|
||||
algorithm: CompressionAlgorithm,
|
||||
backend: ReadCompressionBackend,
|
||||
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync + 'static,
|
||||
{
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
match backend {
|
||||
ReadCompressionBackend::Legacy => Box::new(rustfs_rio::DecompressReader::new(reader, algorithm)),
|
||||
ReadCompressionBackend::V2 => Box::new(rustfs_rio_v2::DecompressReader::new(reader, algorithm)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
{
|
||||
let _ = backend;
|
||||
Box::new(rustfs_rio::DecompressReader::new(reader, algorithm))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decrypt_reader<R>(
|
||||
reader: R,
|
||||
key: [u8; 32],
|
||||
base_nonce: [u8; 12],
|
||||
backend: ReadEncryptionBackend,
|
||||
sequence_number: u32,
|
||||
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync + 'static,
|
||||
{
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
match backend {
|
||||
ReadEncryptionBackend::Legacy => Box::new(rustfs_rio::DecryptReader::new(reader, key, base_nonce)),
|
||||
ReadEncryptionBackend::V2 => {
|
||||
Box::new(rustfs_rio_v2::DecryptReader::new_with_sequence(reader, key, base_nonce, sequence_number))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
{
|
||||
let _ = (backend, sequence_number);
|
||||
Box::new(rustfs_rio::DecryptReader::new(reader, key, base_nonce))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decrypt_reader_with_object_key<R>(
|
||||
reader: R,
|
||||
object_key: [u8; 32],
|
||||
sequence_number: u32,
|
||||
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync + 'static,
|
||||
{
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
Box::new(rustfs_rio_v2::DecryptReader::new_with_object_key_and_sequence(
|
||||
reader,
|
||||
object_key,
|
||||
sequence_number,
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
{
|
||||
let _ = sequence_number;
|
||||
Box::new(rustfs_rio::DecryptReader::new(reader, object_key, [0u8; 12]))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decrypt_multipart_reader<R>(
|
||||
reader: R,
|
||||
key: [u8; 32],
|
||||
base_nonce: [u8; 12],
|
||||
multipart_parts: Vec<usize>,
|
||||
backend: ReadEncryptionBackend,
|
||||
sequence_number: u32,
|
||||
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync + 'static,
|
||||
{
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
match backend {
|
||||
ReadEncryptionBackend::Legacy => {
|
||||
Box::new(rustfs_rio::DecryptReader::new_multipart(reader, key, base_nonce, multipart_parts))
|
||||
}
|
||||
ReadEncryptionBackend::V2 => Box::new(rustfs_rio_v2::DecryptReader::new_multipart_with_sequence(
|
||||
reader,
|
||||
key,
|
||||
base_nonce,
|
||||
multipart_parts,
|
||||
sequence_number,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
{
|
||||
let _ = (backend, sequence_number);
|
||||
Box::new(rustfs_rio::DecryptReader::new_multipart(reader, key, base_nonce, multipart_parts))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decrypt_multipart_reader_with_object_key<R>(
|
||||
reader: R,
|
||||
object_key: [u8; 32],
|
||||
multipart_parts: Vec<usize>,
|
||||
sequence_number: u32,
|
||||
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync + 'static,
|
||||
{
|
||||
#[cfg(feature = "rio-v2")]
|
||||
{
|
||||
Box::new(rustfs_rio_v2::DecryptReader::new_multipart_with_object_key_and_sequence(
|
||||
reader,
|
||||
object_key,
|
||||
multipart_parts,
|
||||
sequence_number,
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
{
|
||||
let _ = sequence_number;
|
||||
Box::new(rustfs_rio::DecryptReader::new_multipart(reader, object_key, [0u8; 12], multipart_parts))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
fn restore_legacy_index_headers(bytes: &[u8]) -> Vec<u8> {
|
||||
if bytes.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
const S2_INDEX_HEADER: &[u8] = b"s2idx\x00";
|
||||
const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s";
|
||||
let mut restored = Vec::with_capacity(4 + S2_INDEX_HEADER.len() + bytes.len() + 4 + S2_INDEX_TRAILER.len());
|
||||
restored.extend_from_slice(&[0x99, 0x2A, 0x4D, 0x18]);
|
||||
restored.extend_from_slice(S2_INDEX_HEADER);
|
||||
restored.extend_from_slice(bytes);
|
||||
|
||||
let total_size = (restored.len() + 4 + S2_INDEX_TRAILER.len()) as u32;
|
||||
restored.extend_from_slice(&total_size.to_le_bytes());
|
||||
restored.extend_from_slice(S2_INDEX_TRAILER);
|
||||
|
||||
let chunk_len = restored.len() - 4;
|
||||
restored[1] = chunk_len as u8;
|
||||
restored[2] = (chunk_len >> 8) as u8;
|
||||
restored[3] = (chunk_len >> 16) as u8;
|
||||
restored
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct WriteEncryption {
|
||||
key_bytes: [u8; 32],
|
||||
mode: WriteEncryptionMode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum WriteEncryptionMode {
|
||||
SinglepartObjectKey,
|
||||
Singlepart {
|
||||
base_nonce: [u8; 12],
|
||||
},
|
||||
MultipartLegacy {
|
||||
base_nonce: [u8; 12],
|
||||
multipart_part_number: usize,
|
||||
},
|
||||
MultipartObjectKey {
|
||||
multipart_part_number: u32,
|
||||
},
|
||||
}
|
||||
|
||||
impl WriteEncryption {
|
||||
pub const fn singlepart_object_key(object_key: [u8; 32]) -> Self {
|
||||
Self {
|
||||
key_bytes: object_key,
|
||||
mode: WriteEncryptionMode::SinglepartObjectKey,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn singlepart(key_bytes: [u8; 32], base_nonce: [u8; 12]) -> Self {
|
||||
Self {
|
||||
key_bytes,
|
||||
mode: WriteEncryptionMode::Singlepart { base_nonce },
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn multipart(key_bytes: [u8; 32], base_nonce: [u8; 12], multipart_part_number: usize) -> Self {
|
||||
Self {
|
||||
key_bytes,
|
||||
mode: WriteEncryptionMode::MultipartLegacy {
|
||||
base_nonce,
|
||||
multipart_part_number,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn multipart_object_key(object_key: [u8; 32], multipart_part_number: u32) -> Self {
|
||||
Self {
|
||||
key_bytes: object_key,
|
||||
mode: WriteEncryptionMode::MultipartObjectKey { multipart_part_number },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct WritePlan {
|
||||
compression: Option<CompressionAlgorithm>,
|
||||
encryption: Option<WriteEncryption>,
|
||||
}
|
||||
|
||||
impl WritePlan {
|
||||
pub const fn new() -> Self {
|
||||
Self {
|
||||
compression: None,
|
||||
encryption: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn with_compression(mut self, algorithm: CompressionAlgorithm) -> Self {
|
||||
self.compression = Some(algorithm);
|
||||
self
|
||||
}
|
||||
|
||||
pub const fn with_encryption(mut self, encryption: WriteEncryption) -> Self {
|
||||
self.encryption = Some(encryption);
|
||||
self
|
||||
}
|
||||
|
||||
pub const fn is_passthrough(&self) -> bool {
|
||||
self.compression.is_none() && self.encryption.is_none()
|
||||
}
|
||||
|
||||
pub fn apply(self, mut reader: HashReader, actual_size: i64) -> std::io::Result<HashReader> {
|
||||
let encrypted = self.encryption.is_some();
|
||||
if let Some(algorithm) = self.compression {
|
||||
reader = HashReader::from_reader(
|
||||
compression_reader(reader, algorithm, encrypted),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(encryption) = self.encryption {
|
||||
reader = match encryption.mode {
|
||||
WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
EncryptReader::new_with_object_key(reader, encryption.key_bytes),
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
EncryptReader::new(reader, encryption.key_bytes, [0u8; 12]),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)?,
|
||||
WriteEncryptionMode::Singlepart { base_nonce } => HashReader::from_reader(
|
||||
EncryptReader::new(reader, encryption.key_bytes, base_nonce),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)?,
|
||||
WriteEncryptionMode::MultipartLegacy {
|
||||
base_nonce,
|
||||
multipart_part_number,
|
||||
} => HashReader::from_reader(
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)?,
|
||||
WriteEncryptionMode::MultipartObjectKey { multipart_part_number } => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
EncryptReader::new_multipart_with_object_key(reader, encryption.key_bytes, multipart_part_number),
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, [0u8; 12], multipart_part_number as usize),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)?,
|
||||
};
|
||||
}
|
||||
|
||||
Ok(reader)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
fn s2_chunk_types(stream: &[u8]) -> Vec<u8> {
|
||||
let mut chunk_types = Vec::new();
|
||||
let mut offset = 0usize;
|
||||
while offset + 4 <= stream.len() {
|
||||
let chunk_type = stream[offset];
|
||||
let chunk_len =
|
||||
(stream[offset + 1] as usize) | ((stream[offset + 2] as usize) << 8) | ((stream[offset + 3] as usize) << 16);
|
||||
chunk_types.push(chunk_type);
|
||||
offset += 4 + chunk_len;
|
||||
}
|
||||
chunk_types
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_plan_passthrough_keeps_plaintext() {
|
||||
let plaintext = b"write-plan-plain".to_vec();
|
||||
let reader = HashReader::from_stream(
|
||||
Cursor::new(plaintext.clone()),
|
||||
plaintext.len() as i64,
|
||||
plaintext.len() as i64,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.expect("create hash reader");
|
||||
|
||||
let mut reader = WritePlan::new()
|
||||
.apply(reader, plaintext.len() as i64)
|
||||
.expect("apply passthrough plan");
|
||||
|
||||
let mut actual = Vec::new();
|
||||
reader.read_to_end(&mut actual).await.expect("read passthrough stream");
|
||||
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_plan_compress_then_encrypt_multipart_roundtrip() {
|
||||
let plaintext = b"abcdefghijklmnopqrstuvwxyz".repeat(128);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
let key_bytes = [0x5Au8; 32];
|
||||
let base_nonce = [0xA5u8; 12];
|
||||
let part_number = 7;
|
||||
|
||||
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
|
||||
.expect("create hash reader");
|
||||
|
||||
let mut transformed = WritePlan::new()
|
||||
.with_compression(CompressionAlgorithm::default())
|
||||
.with_encryption(WriteEncryption::multipart(key_bytes, base_nonce, part_number))
|
||||
.apply(reader, actual_size)
|
||||
.expect("apply transform plan");
|
||||
|
||||
let mut ciphertext = Vec::new();
|
||||
transformed
|
||||
.read_to_end(&mut ciphertext)
|
||||
.await
|
||||
.expect("read transformed ciphertext");
|
||||
|
||||
let decrypt_reader = DecryptReader::new_multipart(Cursor::new(ciphertext), key_bytes, base_nonce, vec![part_number]);
|
||||
let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default());
|
||||
|
||||
let mut actual = Vec::new();
|
||||
decompressed
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("decrypt and decompress transformed stream");
|
||||
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
#[tokio::test]
|
||||
async fn write_plan_supports_singlepart_object_key_encryption_roundtrip() {
|
||||
let plaintext = b"singlepart-object-key".repeat(512);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
let object_key = [0x7Cu8; 32];
|
||||
|
||||
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
|
||||
.expect("create hash reader");
|
||||
|
||||
let mut transformed = WritePlan::new()
|
||||
.with_encryption(WriteEncryption::singlepart_object_key(object_key))
|
||||
.apply(reader, actual_size)
|
||||
.expect("apply singlepart object-key plan");
|
||||
|
||||
let mut encrypted = Vec::new();
|
||||
transformed
|
||||
.read_to_end(&mut encrypted)
|
||||
.await
|
||||
.expect("read encrypted object-key stream");
|
||||
|
||||
let mut decrypted = DecryptReader::new_with_object_key(Cursor::new(encrypted), object_key);
|
||||
let mut actual = Vec::new();
|
||||
decrypted.read_to_end(&mut actual).await.expect("decrypt object-key stream");
|
||||
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
#[tokio::test]
|
||||
async fn write_plan_supports_multipart_object_key_encryption_roundtrip() {
|
||||
let plaintext = b"multipart-object-key-".repeat(4096);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
let object_key = [0x2Du8; 32];
|
||||
let part_number = 3u32;
|
||||
|
||||
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
|
||||
.expect("create hash reader");
|
||||
|
||||
let mut transformed = WritePlan::new()
|
||||
.with_encryption(WriteEncryption::multipart_object_key(object_key, part_number))
|
||||
.apply(reader, actual_size)
|
||||
.expect("apply multipart object-key encryption");
|
||||
|
||||
let mut ciphertext = Vec::new();
|
||||
transformed
|
||||
.read_to_end(&mut ciphertext)
|
||||
.await
|
||||
.expect("read multipart object-key ciphertext");
|
||||
|
||||
let mut actual = Vec::new();
|
||||
DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), object_key, vec![part_number as usize])
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("decrypt multipart object-key ciphertext");
|
||||
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
#[tokio::test]
|
||||
async fn write_plan_rio_v2_compression_emits_s2_stream_and_seekable_index() {
|
||||
let plaintext = b"rustfs-rio-v2-s2-".repeat(600_000);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
|
||||
.expect("create hash reader");
|
||||
|
||||
let mut transformed = WritePlan::new()
|
||||
.with_compression(CompressionAlgorithm::default())
|
||||
.apply(reader, actual_size)
|
||||
.expect("apply compression plan");
|
||||
|
||||
let mut compressed = Vec::new();
|
||||
transformed
|
||||
.read_to_end(&mut compressed)
|
||||
.await
|
||||
.expect("read compressed stream");
|
||||
|
||||
assert!(
|
||||
compressed.starts_with(b"\xff\x06\x00\x00S2sTwO"),
|
||||
"rio_v2 compressed stream must start with the S2 stream identifier"
|
||||
);
|
||||
|
||||
let index = transformed
|
||||
.try_get_index()
|
||||
.cloned()
|
||||
.expect("rio_v2 compressed stream should expose a compression index");
|
||||
let (compressed_offset, uncompressed_offset) = index.find(2 * 1024 * 1024).expect("seek into compression index");
|
||||
|
||||
assert!(compressed_offset > 0, "expected a non-zero compressed offset for the second block");
|
||||
assert!(uncompressed_offset > 0, "expected a non-zero uncompressed offset for the second block");
|
||||
|
||||
let mut decompressed = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
|
||||
let mut actual = Vec::new();
|
||||
decompressed.read_to_end(&mut actual).await.expect("decompress rio_v2 stream");
|
||||
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
#[tokio::test]
|
||||
async fn write_plan_rio_v2_small_compression_skips_index_below_minio_threshold() {
|
||||
let plaintext = b"rustfs-rio-v2-s2-".repeat(32_768);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
|
||||
.expect("create hash reader");
|
||||
|
||||
let mut transformed = WritePlan::new()
|
||||
.with_compression(CompressionAlgorithm::default())
|
||||
.apply(reader, actual_size)
|
||||
.expect("apply compression plan");
|
||||
|
||||
let mut compressed = Vec::new();
|
||||
transformed
|
||||
.read_to_end(&mut compressed)
|
||||
.await
|
||||
.expect("read compressed stream");
|
||||
|
||||
assert!(
|
||||
transformed.try_get_index().is_none(),
|
||||
"rio_v2 should match MinIO and skip compression indexes for small objects"
|
||||
);
|
||||
|
||||
let mut decompressed = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
|
||||
let mut actual = Vec::new();
|
||||
decompressed.read_to_end(&mut actual).await.expect("decompress rio_v2 stream");
|
||||
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
#[tokio::test]
|
||||
async fn rio_v2_singlepart_encrypt_decrypt_roundtrip_preserves_small_compressed_stream() {
|
||||
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
|
||||
let key_bytes = [0x33u8; 32];
|
||||
let base_nonce = [0x55u8; 12];
|
||||
|
||||
let mut compressed = Vec::new();
|
||||
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
|
||||
.read_to_end(&mut compressed)
|
||||
.await
|
||||
.expect("compress plaintext");
|
||||
|
||||
let mut encrypted = Vec::new();
|
||||
EncryptReader::new(Cursor::new(compressed), key_bytes, base_nonce)
|
||||
.read_to_end(&mut encrypted)
|
||||
.await
|
||||
.expect("encrypt compressed stream");
|
||||
|
||||
let decrypt_reader = DecryptReader::new(Cursor::new(encrypted), key_bytes, base_nonce);
|
||||
let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default());
|
||||
let mut actual = Vec::new();
|
||||
decompressed
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("decrypt and decompress small stream");
|
||||
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
#[tokio::test]
|
||||
async fn rio_v2_compress_then_encrypt_adds_s2_padding_frames() {
|
||||
let plaintext = b"padding-check-".repeat(4097);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
let key_bytes = [0x1Bu8; 32];
|
||||
let base_nonce = [0xC4u8; 12];
|
||||
|
||||
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
|
||||
.expect("create hash reader");
|
||||
|
||||
let mut transformed = WritePlan::new()
|
||||
.with_compression(CompressionAlgorithm::default())
|
||||
.with_encryption(WriteEncryption::singlepart(key_bytes, base_nonce))
|
||||
.apply(reader, actual_size)
|
||||
.expect("apply transform plan");
|
||||
|
||||
let mut ciphertext = Vec::new();
|
||||
transformed
|
||||
.read_to_end(&mut ciphertext)
|
||||
.await
|
||||
.expect("read transformed ciphertext");
|
||||
|
||||
let mut decrypted_compressed = Vec::new();
|
||||
DecryptReader::new(Cursor::new(ciphertext), key_bytes, base_nonce)
|
||||
.read_to_end(&mut decrypted_compressed)
|
||||
.await
|
||||
.expect("decrypt compressed stream");
|
||||
|
||||
assert_eq!(decrypted_compressed.len() % ENCRYPTED_S2_PADDING_MULTIPLE, 0);
|
||||
|
||||
let chunk_types = s2_chunk_types(&decrypted_compressed);
|
||||
assert!(
|
||||
chunk_types.contains(&0xfe),
|
||||
"rio_v2 compressed+encrypted streams must include S2 padding frames before encryption"
|
||||
);
|
||||
|
||||
let mut actual = Vec::new();
|
||||
DecompressReader::new(Cursor::new(decrypted_compressed), CompressionAlgorithm::default())
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("decompress padded stream");
|
||||
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
#[tokio::test]
|
||||
async fn rio_v2_decompress_reader_returns_bytes_on_first_read() {
|
||||
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
|
||||
|
||||
let mut compressed = Vec::new();
|
||||
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
|
||||
.read_to_end(&mut compressed)
|
||||
.await
|
||||
.expect("compress plaintext");
|
||||
|
||||
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
|
||||
let mut buf = [0u8; 64];
|
||||
let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk");
|
||||
|
||||
assert!(n > 0);
|
||||
assert_eq!(&buf[..n], plaintext.as_slice());
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
#[tokio::test]
|
||||
async fn rio_v2_decompress_reader_returns_bytes_on_first_large_read() {
|
||||
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
|
||||
|
||||
let mut compressed = Vec::new();
|
||||
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
|
||||
.read_to_end(&mut compressed)
|
||||
.await
|
||||
.expect("compress plaintext");
|
||||
|
||||
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
|
||||
let mut buf = [0u8; 8192];
|
||||
let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk");
|
||||
|
||||
assert!(n > 0);
|
||||
assert_eq!(&buf[..n], plaintext.as_slice());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user