gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 11:56:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(rio): rio_v2 is compatible with minio for storing data. (#3115)

Browse Source 
* Set up a compatibility layer for replacing old Rio components with new ones.

* fix(rio). compress range

* feat(rio). Add the experimental feature rio_v2 to support minio data at the binary level.

* feat(rio_v2): add sse-c test

* test compression component

* simple fix

* fix minlz encode

* fix metadata

* fix kms key cache error

* Update launch.json

* ci: set nix crate download user agent

* fix: gate obs pyroscope backend

* ignore minio test

* fix encrypt check

* fix

* fix

* fix

* Update object_usecase.rs

* Update ci.yml

* fix

* ci add rio-v2 test

* fix

* ci fix

* fix

* Reconstructed into a more reasonable compatibility mode

* fix

* fix

---------

Signed-off-by: houseme <housemecn@gmail.com>
Signed-off-by: 唐小鸭 <tangtang1251@qq.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
唐小鸭
2026-06-08 19:59:14 +08:00
committed by GitHub
parent 9504dff595
commit f7724d223b
47 changed files with 8742 additions and 682 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/rio-v2/Cargo.toml
+48
View File
@@ -0,0 +1,48 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-rio-v2"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
homepage.workspace = true
description = "Feature-gated rio_v2 compatibility facade for incremental RustFS migration."
keywords = ["asynchronous", "io", "compatibility", "rustfs", "minio"]
categories = ["web-programming", "development-tools", "asynchronous"]
documentation = "https://docs.rs/rustfs-rio-v2/latest/rustfs_rio_v2/"
[lints]
workspace = true
[dependencies]
aes-gcm.workspace = true
bytes.workspace = true
hex.workspace = true
hmac.workspace = true
minlz = "1.1.0"
pin-project-lite.workspace = true
rand.workspace = true
rustfs-rio.workspace = true
rustfs-utils.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
tokio.workspace = true
[dev-dependencies]
rustfs-filemeta.workspace = true
walkdir.workspace = true
crates/rio-v2/src/compress_reader.rs
+755
View File
@@ -0,0 +1,755 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use minlz::{Encoder as MinlzEncoder, crc::crc, decode};
use pin_project_lite::pin_project;
use rand::RngExt;
use rustfs_rio::{EtagResolvable, HashReaderDetector, HashReaderMut, Index, TryGetIndex};
use rustfs_utils::CompressionAlgorithm;
use std::cmp::min;
use std::fmt;
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
const MAGIC_CHUNK: &[u8] = b"\xff\x06\x00\x00S2sTwO";
const MAGIC_CHUNK_SNAPPY: &[u8] = b"\xff\x06\x00\x00sNaPpY";
const CHUNK_TYPE_COMPRESSED_DATA: u8 = 0x00;
const CHUNK_TYPE_UNCOMPRESSED_DATA: u8 = 0x01;
const CHUNK_TYPE_INDEX: u8 = 0x99;
const CHUNK_TYPE_PADDING: u8 = 0xfe;
const CHUNK_TYPE_STREAM_IDENTIFIER: u8 = 0xff;
const DEFAULT_BLOCK_SIZE: usize = 1 << 20;
const MAX_CHUNK_SIZE: usize = (1 << 24) - 1;
const CHECKSUM_SIZE: usize = 4;
const CHUNK_HEADER_LEN: usize = 4;
const ENCRYPTED_PADDING_MULTIPLE: usize = 256;
const MIN_INDEX_SIZE: usize = 8 << 20;
pin_project! {
#[derive(Debug)]
pub struct CompressReader<R> {
#[pin]
inner: R,
buffer: Vec<u8>,
pos: usize,
done: bool,
block_size: usize,
index: Index,
written: usize,
uncompressed_written: usize,
temp_buffer: Vec<u8>,
read_buffer: Vec<u8>,
wrote_stream_header: bool,
padding_multiple: Option<usize>,
block_encoder: S2BlockEncoder,
}
}
struct S2BlockEncoder {
inner: MinlzEncoder,
}
impl S2BlockEncoder {
fn new() -> Self {
Self {
inner: MinlzEncoder::new(),
}
}
fn encode(&mut self, uncompressed: &[u8]) -> Vec<u8> {
self.inner.encode(uncompressed)
}
}
impl fmt::Debug for S2BlockEncoder {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("S2BlockEncoder").finish_non_exhaustive()
}
}
impl<R> CompressReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
pub fn new(inner: R, _compression_algorithm: CompressionAlgorithm) -> Self {
Self::with_block_size(inner, DEFAULT_BLOCK_SIZE, CompressionAlgorithm::default())
}
pub fn with_block_size(inner: R, block_size: usize, _compression_algorithm: CompressionAlgorithm) -> Self {
Self {
inner,
buffer: Vec::new(),
pos: 0,
done: false,
block_size,
index: Index::new(),
written: 0,
uncompressed_written: 0,
temp_buffer: Vec::with_capacity(block_size),
read_buffer: vec![0u8; block_size],
wrote_stream_header: false,
padding_multiple: None,
block_encoder: S2BlockEncoder::new(),
}
}
pub fn with_encrypted_padding(inner: R, _compression_algorithm: CompressionAlgorithm) -> Self {
let mut reader = Self::new(inner, CompressionAlgorithm::default());
reader.padding_multiple = Some(ENCRYPTED_PADDING_MULTIPLE);
reader
}
}
impl<R> TryGetIndex for CompressReader<R> {
fn try_get_index(&self) -> Option<&Index> {
(self.uncompressed_written > MIN_INDEX_SIZE).then_some(&self.index)
}
}
impl<R> AsyncRead for CompressReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let mut this = self.project();
if *this.pos < this.buffer.len() {
let to_copy = min(buf.remaining(), this.buffer.len() - *this.pos);
buf.put_slice(&this.buffer[*this.pos..*this.pos + to_copy]);
*this.pos += to_copy;
if *this.pos == this.buffer.len() {
this.buffer.clear();
*this.pos = 0;
}
return Poll::Ready(Ok(()));
}
if *this.done {
return Poll::Ready(Ok(()));
}
while this.temp_buffer.len() < *this.block_size {
let remaining = *this.block_size - this.temp_buffer.len();
let mut read_buf = ReadBuf::new(&mut this.read_buffer[..remaining]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => {
return Poll::Pending;
}
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
break;
}
this.temp_buffer.extend_from_slice(read_buf.filled());
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
if this.temp_buffer.is_empty() {
if let Some(padding_multiple) = *this.padding_multiple
&& let Some(padding_chunk) = build_padding_chunk(*this.written, padding_multiple)?
{
*this.written += padding_chunk.len();
this.index.total_compressed = *this.written as i64;
*this.buffer = padding_chunk;
*this.pos = 0;
*this.done = true;
let to_copy = min(buf.remaining(), this.buffer.len());
buf.put_slice(&this.buffer[..to_copy]);
*this.pos += to_copy;
if *this.pos == this.buffer.len() {
this.buffer.clear();
*this.pos = 0;
}
return Poll::Ready(Ok(()));
}
*this.done = true;
return Poll::Ready(Ok(()));
}
let mut out = Vec::new();
if !*this.wrote_stream_header {
out.extend_from_slice(MAGIC_CHUNK);
*this.written += MAGIC_CHUNK.len();
*this.wrote_stream_header = true;
}
if let Err(err) = this.index.add(*this.written as i64, *this.uncompressed_written as i64) {
return Poll::Ready(Err(err));
}
let block = build_s2_chunk(this.temp_buffer.as_slice(), this.block_encoder)?;
*this.uncompressed_written += this.temp_buffer.len();
*this.written += block.len();
this.index.total_uncompressed = *this.uncompressed_written as i64;
this.index.total_compressed = *this.written as i64;
out.extend_from_slice(&block);
this.temp_buffer.clear();
*this.buffer = out;
*this.pos = 0;
let to_copy = min(buf.remaining(), this.buffer.len());
buf.put_slice(&this.buffer[..to_copy]);
*this.pos += to_copy;
if *this.pos == this.buffer.len() {
this.buffer.clear();
*this.pos = 0;
}
Poll::Ready(Ok(()))
}
}
impl<R> EtagResolvable for CompressReader<R>
where
R: EtagResolvable,
{
fn try_resolve_etag(&mut self) -> Option<String> {
self.inner.try_resolve_etag()
}
}
impl<R> HashReaderDetector for CompressReader<R>
where
R: HashReaderDetector,
{
fn is_hash_reader(&self) -> bool {
self.inner.is_hash_reader()
}
fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> {
self.inner.as_hash_reader_mut()
}
}
pin_project! {
#[derive(Debug)]
pub struct DecompressReader<R> {
#[pin]
inner: R,
buffer: Vec<u8>,
buffer_pos: usize,
finished: bool,
header_buf: [u8; CHUNK_HEADER_LEN],
header_read: usize,
chunk_type: u8,
chunk_buf: Vec<u8>,
chunk_len: usize,
chunk_read: usize,
reading_chunk: bool,
stream_initialized: bool,
}
}
impl<R> DecompressReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
pub fn new(inner: R, _compression_algorithm: CompressionAlgorithm) -> Self {
Self {
inner,
buffer: Vec::new(),
buffer_pos: 0,
finished: false,
header_buf: [0u8; CHUNK_HEADER_LEN],
header_read: 0,
chunk_type: 0,
chunk_buf: Vec::new(),
chunk_len: 0,
chunk_read: 0,
reading_chunk: false,
stream_initialized: false,
}
}
}
impl<R> AsyncRead for DecompressReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let mut this = self.project();
if *this.buffer_pos < this.buffer.len() {
let to_copy = min(buf.remaining(), this.buffer.len() - *this.buffer_pos);
buf.put_slice(&this.buffer[*this.buffer_pos..*this.buffer_pos + to_copy]);
*this.buffer_pos += to_copy;
if *this.buffer_pos == this.buffer.len() {
this.buffer.clear();
*this.buffer_pos = 0;
}
return Poll::Ready(Ok(()));
}
loop {
if *this.finished {
return Poll::Ready(Ok(()));
}
if !*this.reading_chunk {
while *this.header_read < CHUNK_HEADER_LEN {
let mut read_buf = ReadBuf::new(&mut this.header_buf[*this.header_read..]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
if *this.header_read == 0 {
*this.finished = true;
return Poll::Ready(Ok(()));
}
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF while reading S2 chunk header",
)));
}
*this.header_read += n;
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
*this.chunk_type = this.header_buf[0];
*this.chunk_len =
(this.header_buf[1] as usize) | ((this.header_buf[2] as usize) << 8) | ((this.header_buf[3] as usize) << 16);
*this.header_read = 0;
if this.chunk_buf.len() < *this.chunk_len {
this.chunk_buf.resize(*this.chunk_len, 0);
}
*this.chunk_read = 0;
*this.reading_chunk = true;
}
while *this.chunk_read < *this.chunk_len {
let mut read_buf = ReadBuf::new(&mut this.chunk_buf[*this.chunk_read..*this.chunk_len]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF while reading S2 chunk body",
)));
}
*this.chunk_read += n;
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
let chunk = &this.chunk_buf[..*this.chunk_len];
*this.reading_chunk = false;
match *this.chunk_type {
CHUNK_TYPE_STREAM_IDENTIFIER => {
if chunk != &MAGIC_CHUNK[CHUNK_HEADER_LEN..] && chunk != &MAGIC_CHUNK_SNAPPY[CHUNK_HEADER_LEN..] {
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "invalid S2 stream identifier")));
}
*this.stream_initialized = true;
continue;
}
CHUNK_TYPE_COMPRESSED_DATA => {
*this.stream_initialized = true;
let decompressed = decode_chunk(chunk, true)?;
*this.buffer = decompressed;
}
CHUNK_TYPE_UNCOMPRESSED_DATA => {
*this.stream_initialized = true;
let decompressed = decode_chunk(chunk, false)?;
*this.buffer = decompressed;
}
CHUNK_TYPE_INDEX | CHUNK_TYPE_PADDING | 0x80..=0xfd => {
*this.stream_initialized = true;
continue;
}
_ => {
if !*this.stream_initialized && *this.chunk_type != CHUNK_TYPE_COMPRESSED_DATA {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown S2 chunk type: 0x{:02x}", *this.chunk_type),
)));
}
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown S2 chunk type: 0x{:02x}", *this.chunk_type),
)));
}
}
*this.buffer_pos = 0;
let to_copy = min(buf.remaining(), this.buffer.len());
buf.put_slice(&this.buffer[..to_copy]);
*this.buffer_pos += to_copy;
if *this.buffer_pos == this.buffer.len() {
this.buffer.clear();
*this.buffer_pos = 0;
}
return Poll::Ready(Ok(()));
}
}
}
impl<R> EtagResolvable for DecompressReader<R>
where
R: EtagResolvable,
{
fn try_resolve_etag(&mut self) -> Option<String> {
self.inner.try_resolve_etag()
}
}
impl<R> HashReaderDetector for DecompressReader<R>
where
R: HashReaderDetector,
{
fn is_hash_reader(&self) -> bool {
self.inner.is_hash_reader()
}
fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> {
self.inner.as_hash_reader_mut()
}
}
fn build_s2_chunk(uncompressed: &[u8], encoder: &mut S2BlockEncoder) -> io::Result<Vec<u8>> {
let compressed = encode_block(uncompressed, encoder);
let checksum = crc(uncompressed);
let dst_limit = uncompressed.len().saturating_sub(uncompressed.len() / 32).saturating_sub(5);
let (chunk_type, payload) = if compressed.len() <= dst_limit {
(CHUNK_TYPE_COMPRESSED_DATA, compressed)
} else {
(CHUNK_TYPE_UNCOMPRESSED_DATA, uncompressed.to_vec())
};
let chunk_len = payload.len() + CHECKSUM_SIZE;
if chunk_len > MAX_CHUNK_SIZE {
return Err(io::Error::new(io::ErrorKind::InvalidData, "S2 chunk exceeds 24-bit framing limit"));
}
let mut out = Vec::with_capacity(CHUNK_HEADER_LEN + chunk_len);
out.push(chunk_type);
out.push((chunk_len & 0xff) as u8);
out.push(((chunk_len >> 8) & 0xff) as u8);
out.push(((chunk_len >> 16) & 0xff) as u8);
out.extend_from_slice(&checksum.to_le_bytes());
out.extend_from_slice(&payload);
Ok(out)
}
fn encode_block(uncompressed: &[u8], encoder: &mut S2BlockEncoder) -> Vec<u8> {
encoder.encode(uncompressed)
}
fn build_padding_chunk(current_size: usize, padding_multiple: usize) -> io::Result<Option<Vec<u8>>> {
if padding_multiple == 0 || current_size.is_multiple_of(padding_multiple) {
return Ok(None);
}
let padding_len = (padding_multiple - ((current_size + CHUNK_HEADER_LEN) % padding_multiple)) % padding_multiple;
if padding_len > MAX_CHUNK_SIZE {
return Err(io::Error::new(io::ErrorKind::InvalidData, "S2 padding exceeds 24-bit framing limit"));
}
let mut out = Vec::with_capacity(CHUNK_HEADER_LEN + padding_len);
out.push(CHUNK_TYPE_PADDING);
out.push((padding_len & 0xff) as u8);
out.push(((padding_len >> 8) & 0xff) as u8);
out.push(((padding_len >> 16) & 0xff) as u8);
if padding_len > 0 {
let mut padding = vec![0u8; padding_len];
rand::rng().fill(padding.as_mut_slice());
out.extend_from_slice(&padding);
}
Ok(Some(out))
}
fn decode_chunk(chunk: &[u8], compressed: bool) -> io::Result<Vec<u8>> {
if chunk.len() < CHECKSUM_SIZE {
return Err(io::Error::new(io::ErrorKind::InvalidData, "S2 chunk smaller than checksum header"));
}
let expected_crc = u32::from_le_bytes(chunk[..CHECKSUM_SIZE].try_into().expect("checksum header"));
let payload = &chunk[CHECKSUM_SIZE..];
let decompressed = if compressed {
decode(payload).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("S2 decode error: {err}")))?
} else {
payload.to_vec()
};
let actual_crc = crc(&decompressed);
if actual_crc != expected_crc {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"S2 CRC mismatch: expected={expected_crc:08x} actual={actual_crc:08x} compressed={compressed} payload_len={} decompressed_len={}",
payload.len(),
decompressed.len()
),
));
}
Ok(decompressed)
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::AsyncReadExt;
struct PendingAfterBytes<R> {
inner: R,
max_chunk: usize,
pending_next: bool,
}
impl<R> PendingAfterBytes<R> {
fn new(inner: R, max_chunk: usize) -> Self {
Self {
inner,
max_chunk,
pending_next: false,
}
}
}
impl<R: AsyncRead + Unpin> AsyncRead for PendingAfterBytes<R> {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
if self.pending_next {
self.pending_next = false;
cx.waker().wake_by_ref();
return Poll::Pending;
}
let allowed = self.max_chunk.min(buf.remaining());
if allowed == 0 {
return Poll::Ready(Ok(()));
}
let mut scratch = vec![0u8; allowed];
let mut limited = ReadBuf::new(&mut scratch);
match Pin::new(&mut self.inner).poll_read(cx, &mut limited) {
Poll::Pending => Poll::Pending,
Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
Poll::Ready(Ok(())) => {
let filled = limited.filled();
if !filled.is_empty() {
buf.put_slice(filled);
self.pending_next = true;
}
Poll::Ready(Ok(()))
}
}
}
}
fn s2_chunk_types(stream: &[u8]) -> Vec<u8> {
let mut chunk_types = Vec::new();
let mut offset = 0usize;
while offset + CHUNK_HEADER_LEN <= stream.len() {
let chunk_type = stream[offset];
let chunk_len =
(stream[offset + 1] as usize) | ((stream[offset + 2] as usize) << 8) | ((stream[offset + 3] as usize) << 16);
chunk_types.push(chunk_type);
offset += CHUNK_HEADER_LEN + chunk_len;
}
chunk_types
}
#[test]
fn minlz_encoded_payload_decodes_with_minlz() {
let plaintext = b"compressible-rio-v2-block-".repeat(4096);
let mut encoder = S2BlockEncoder::new();
let compressed = encode_block(&plaintext, &mut encoder);
let decoded = decode(&compressed).expect("decode payload");
assert_eq!(decoded, plaintext);
}
#[tokio::test]
async fn s2_compress_reader_roundtrip() {
let plaintext = b"hello-rio-v2-s2-".repeat(32_768);
let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
assert!(compressed.starts_with(MAGIC_CHUNK));
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn s2_compress_reader_roundtrip_near_erasure_boundary() {
let size = 4 * 1024 * 1024 - 97;
let plaintext = pseudo_random_bytes(size);
let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
fn pseudo_random_bytes(size: usize) -> Vec<u8> {
(0..size)
.scan(0x9e37_79b9_7f4a_7c15u64, |state, _| {
*state ^= *state << 7;
*state ^= *state >> 9;
*state = state.wrapping_mul(0xbf58_476d_1ce4_e5b9);
Some((*state >> 32) as u8)
})
.collect()
}
#[tokio::test]
async fn s2_compress_reader_roundtrip_large_random() {
let plaintext = pseudo_random_bytes(8 * 1024 * 1024 + 123);
let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn s2_compress_reader_roundtrip_with_pending_source() {
let plaintext = pseudo_random_bytes(2 * 1024 * 1024 + 17);
let pending_reader = PendingAfterBytes::new(Cursor::new(plaintext.clone()), 257);
let mut reader = CompressReader::new(pending_reader, CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn s2_decompress_reader_returns_bytes_on_first_read() {
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
let mut compressed = Vec::new();
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
.read_to_end(&mut compressed)
.await
.expect("compress plaintext");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut buf = [0u8; 64];
let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk");
assert!(n > 0);
assert_eq!(&buf[..n], plaintext.as_slice());
}
#[tokio::test]
async fn s2_decompress_reader_resumes_chunk_body_after_pending() {
let plaintext = pseudo_random_bytes(1024 * 1024 + 123);
let mut compressed = Vec::new();
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
.read_to_end(&mut compressed)
.await
.expect("compress plaintext");
let pending_reader = PendingAfterBytes::new(Cursor::new(compressed), 257);
let mut decompressor = DecompressReader::new(pending_reader, CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn s2_decompress_reader_handles_concatenated_streams_after_pending() {
let first = pseudo_random_bytes(16 * 1024 * 1024);
let second = pseudo_random_bytes(12 * 1024 * 1024 + 123);
let mut first_compressed = Vec::new();
CompressReader::new(Cursor::new(first.clone()), CompressionAlgorithm::default())
.read_to_end(&mut first_compressed)
.await
.expect("compress first stream");
let mut second_compressed = Vec::new();
CompressReader::new(Cursor::new(second.clone()), CompressionAlgorithm::default())
.read_to_end(&mut second_compressed)
.await
.expect("compress second stream");
first_compressed.extend_from_slice(&second_compressed);
let pending_reader = PendingAfterBytes::new(Cursor::new(first_compressed), 4096);
let mut decompressor = DecompressReader::new(pending_reader, CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
let mut expected = first;
expected.extend_from_slice(&second);
assert_eq!(actual, expected);
}
#[tokio::test]
async fn s2_compress_reader_with_encrypted_padding_emits_padding_frame() {
let plaintext = b"encrypted-padding-check-".repeat(8192);
let mut reader = CompressReader::with_encrypted_padding(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
assert_eq!(compressed.len() % ENCRYPTED_PADDING_MULTIPLE, 0);
assert!(s2_chunk_types(&compressed).contains(&CHUNK_TYPE_PADDING));
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn s2_compress_reader_skips_index_for_small_streams() {
let plaintext = b"index-threshold-check-".repeat(16_384);
let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
assert!(reader.try_get_index().is_none());
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
}
crates/rio-v2/src/encrypt_reader.rs
+759
View File
@@ -0,0 +1,759 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use aes_gcm::aead::Aead;
use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
use hmac::{Hmac, Mac};
use pin_project_lite::pin_project;
use rand::RngExt;
use rustfs_rio::{EtagResolvable, HashReaderDetector, HashReaderMut, Index, TryGetIndex, multipart_part_nonce};
use sha2::Sha256;
use std::cmp::min;
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
const DARE_VERSION_20: u8 = 0x20;
const DARE_CIPHER_AES_256_GCM: u8 = 0x00;
const DARE_HEADER_SIZE: usize = 16;
const DARE_TAG_SIZE: usize = 16;
const DARE_PAYLOAD_SIZE: usize = 64 * 1024;
type HmacSha256 = Hmac<Sha256>;
#[derive(Clone, Copy)]
enum MultipartKeySource {
LegacyNonce { base_nonce: [u8; 12] },
ObjectKey { object_key: [u8; 32] },
}
pin_project! {
pub struct EncryptReader<R> {
#[pin]
inner: R,
cipher: Aes256Gcm,
base_nonce: [u8; 12],
sequence_number: u32,
temp_buffer: Vec<u8>,
read_buffer: Vec<u8>,
output_buffer: Vec<u8>,
output_pos: usize,
finished: bool,
}
}
impl<R> EncryptReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
pub fn new_with_object_key(inner: R, object_key: [u8; 32]) -> Self {
Self::new(inner, object_key, random_stream_nonce())
}
pub fn new(inner: R, key: [u8; 32], nonce: [u8; 12]) -> Self {
Self::new_with_sequence(inner, key, nonce, 0)
}
pub fn new_with_sequence(inner: R, key: [u8; 32], nonce: [u8; 12], sequence_number: u32) -> Self {
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"),
base_nonce: nonce,
sequence_number,
temp_buffer: Vec::with_capacity(DARE_PAYLOAD_SIZE + 1),
read_buffer: vec![0u8; DARE_PAYLOAD_SIZE + 1],
output_buffer: Vec::new(),
output_pos: 0,
finished: false,
}
}
pub fn new_multipart(inner: R, key: [u8; 32], base_nonce: [u8; 12], part_number: usize) -> Self {
Self::new(inner, key, multipart_part_nonce(base_nonce, part_number))
}
pub fn new_multipart_with_object_key(inner: R, object_key: [u8; 32], part_number: u32) -> Self {
Self::new(inner, derive_part_key(object_key, part_number), random_stream_nonce())
}
pub fn new_multipart_with_sequence(
inner: R,
key: [u8; 32],
base_nonce: [u8; 12],
part_number: usize,
sequence_number: u32,
) -> Self {
Self::new_with_sequence(inner, key, multipart_part_nonce(base_nonce, part_number), sequence_number)
}
}
impl<R> AsyncRead for EncryptReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let mut this = self.project();
if *this.output_pos < this.output_buffer.len() {
let to_copy = min(buf.remaining(), this.output_buffer.len() - *this.output_pos);
buf.put_slice(&this.output_buffer[*this.output_pos..*this.output_pos + to_copy]);
*this.output_pos += to_copy;
if *this.output_pos == this.output_buffer.len() {
this.output_buffer.clear();
*this.output_pos = 0;
}
return Poll::Ready(Ok(()));
}
if *this.finished {
return Poll::Ready(Ok(()));
}
loop {
if this.temp_buffer.len() > DARE_PAYLOAD_SIZE {
let package = build_dare_package(
this.cipher,
*this.sequence_number,
*this.base_nonce,
&this.temp_buffer[..DARE_PAYLOAD_SIZE],
false,
)?;
let carry = this.temp_buffer.split_off(DARE_PAYLOAD_SIZE);
*this.temp_buffer = carry;
*this.sequence_number = this.sequence_number.wrapping_add(1);
*this.output_buffer = package;
break;
}
let remaining = DARE_PAYLOAD_SIZE + 1 - this.temp_buffer.len();
let mut read_buf = ReadBuf::new(&mut this.read_buffer[..remaining]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => {
if this.temp_buffer.len() > DARE_PAYLOAD_SIZE {
continue;
}
return Poll::Pending;
}
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
if this.temp_buffer.is_empty() {
*this.finished = true;
return Poll::Ready(Ok(()));
}
let package =
build_dare_package(this.cipher, *this.sequence_number, *this.base_nonce, this.temp_buffer, true)?;
this.temp_buffer.clear();
*this.sequence_number = this.sequence_number.wrapping_add(1);
*this.output_buffer = package;
*this.finished = true;
break;
}
this.temp_buffer.extend_from_slice(read_buf.filled());
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
let to_copy = min(buf.remaining(), this.output_buffer.len());
buf.put_slice(&this.output_buffer[..to_copy]);
*this.output_pos += to_copy;
if *this.output_pos == this.output_buffer.len() {
this.output_buffer.clear();
*this.output_pos = 0;
}
Poll::Ready(Ok(()))
}
}
impl<R> EtagResolvable for EncryptReader<R>
where
R: EtagResolvable,
{
fn try_resolve_etag(&mut self) -> Option<String> {
self.inner.try_resolve_etag()
}
}
impl<R> HashReaderDetector for EncryptReader<R>
where
R: HashReaderDetector,
{
fn is_hash_reader(&self) -> bool {
self.inner.is_hash_reader()
}
fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> {
self.inner.as_hash_reader_mut()
}
}
impl<R> TryGetIndex for EncryptReader<R>
where
R: TryGetIndex,
{
fn try_get_index(&self) -> Option<&Index> {
self.inner.try_get_index()
}
}
pin_project! {
pub struct DecryptReader<R> {
#[pin]
inner: R,
cipher: Aes256Gcm,
expected_base_nonce: Option<[u8; 12]>,
sequence_number: u32,
multipart_parts: Vec<usize>,
current_part_index: usize,
multipart_mode: bool,
multipart_key_source: Option<MultipartKeySource>,
header_buf: [u8; DARE_HEADER_SIZE],
header_read: usize,
ciphertext_buf: Vec<u8>,
ciphertext_len: usize,
ciphertext_read: usize,
ref_nonce: Option<[u8; 12]>,
finalized: bool,
finished: bool,
output_buffer: Vec<u8>,
output_pos: usize,
}
}
impl<R> DecryptReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
pub fn new_with_object_key(inner: R, object_key: [u8; 32]) -> Self {
Self::new_with_object_key_and_sequence(inner, object_key, 0)
}
pub fn new(inner: R, key: [u8; 32], nonce: [u8; 12]) -> Self {
Self::new_with_sequence(inner, key, nonce, 0)
}
pub fn new_with_object_key_and_sequence(inner: R, object_key: [u8; 32], sequence_number: u32) -> Self {
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&object_key).expect("valid AES-256-GCM key"),
expected_base_nonce: None,
sequence_number,
multipart_parts: Vec::new(),
current_part_index: 0,
multipart_mode: false,
multipart_key_source: None,
header_buf: [0u8; DARE_HEADER_SIZE],
header_read: 0,
ciphertext_buf: Vec::new(),
ciphertext_len: 0,
ciphertext_read: 0,
ref_nonce: None,
finalized: false,
finished: false,
output_buffer: Vec::new(),
output_pos: 0,
}
}
pub fn new_with_sequence(inner: R, key: [u8; 32], nonce: [u8; 12], sequence_number: u32) -> Self {
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"),
expected_base_nonce: Some(nonce),
sequence_number,
multipart_parts: Vec::new(),
current_part_index: 0,
multipart_mode: false,
multipart_key_source: None,
header_buf: [0u8; DARE_HEADER_SIZE],
header_read: 0,
ciphertext_buf: Vec::new(),
ciphertext_len: 0,
ciphertext_read: 0,
ref_nonce: None,
finalized: false,
finished: false,
output_buffer: Vec::new(),
output_pos: 0,
}
}
pub fn new_multipart(inner: R, key: [u8; 32], base_nonce: [u8; 12], multipart_parts: Vec<usize>) -> Self {
Self::new_multipart_with_sequence(inner, key, base_nonce, multipart_parts, 0)
}
pub fn new_multipart_with_object_key(inner: R, object_key: [u8; 32], multipart_parts: Vec<usize>) -> Self {
Self::new_multipart_with_object_key_and_sequence(inner, object_key, multipart_parts, 0)
}
pub fn new_multipart_with_sequence(
inner: R,
key: [u8; 32],
base_nonce: [u8; 12],
multipart_parts: Vec<usize>,
sequence_number: u32,
) -> Self {
let first_part = multipart_parts.first().copied().unwrap_or(1);
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"),
expected_base_nonce: Some(multipart_part_nonce(base_nonce, first_part)),
sequence_number,
multipart_parts,
current_part_index: 0,
multipart_mode: true,
multipart_key_source: Some(MultipartKeySource::LegacyNonce { base_nonce }),
header_buf: [0u8; DARE_HEADER_SIZE],
header_read: 0,
ciphertext_buf: Vec::new(),
ciphertext_len: 0,
ciphertext_read: 0,
ref_nonce: None,
finalized: false,
finished: false,
output_buffer: Vec::new(),
output_pos: 0,
}
}
pub fn new_multipart_with_object_key_and_sequence(
inner: R,
object_key: [u8; 32],
multipart_parts: Vec<usize>,
sequence_number: u32,
) -> Self {
let first_part = multipart_parts.first().copied().unwrap_or(1);
let first_key = derive_part_key(object_key, first_part as u32);
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&first_key).expect("valid AES-256-GCM key"),
expected_base_nonce: None,
sequence_number,
multipart_parts,
current_part_index: 0,
multipart_mode: true,
multipart_key_source: Some(MultipartKeySource::ObjectKey { object_key }),
header_buf: [0u8; DARE_HEADER_SIZE],
header_read: 0,
ciphertext_buf: Vec::new(),
ciphertext_len: 0,
ciphertext_read: 0,
ref_nonce: None,
finalized: false,
finished: false,
output_buffer: Vec::new(),
output_pos: 0,
}
}
}
impl<R> AsyncRead for DecryptReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let mut this = self.project();
if *this.output_pos < this.output_buffer.len() {
let to_copy = min(buf.remaining(), this.output_buffer.len() - *this.output_pos);
buf.put_slice(&this.output_buffer[*this.output_pos..*this.output_pos + to_copy]);
*this.output_pos += to_copy;
if *this.output_pos == this.output_buffer.len() {
this.output_buffer.clear();
*this.output_pos = 0;
}
return Poll::Ready(Ok(()));
}
loop {
if *this.finished {
return Poll::Ready(Ok(()));
}
if *this.finalized {
if !this.multipart_parts.is_empty() && *this.current_part_index + 1 < this.multipart_parts.len() {
let next_part = this.multipart_parts[*this.current_part_index + 1];
*this.current_part_index += 1;
match this.multipart_key_source.as_ref().copied() {
Some(MultipartKeySource::LegacyNonce { base_nonce }) => {
*this.expected_base_nonce = Some(multipart_part_nonce(base_nonce, next_part));
}
Some(MultipartKeySource::ObjectKey { object_key }) => {
let part_key = derive_part_key(object_key, next_part as u32);
*this.cipher = Aes256Gcm::new_from_slice(&part_key).expect("valid AES-256-GCM key");
*this.expected_base_nonce = None;
}
None => {}
}
*this.sequence_number = 0;
*this.ref_nonce = None;
*this.finalized = false;
continue;
}
*this.finished = true;
return Poll::Ready(Ok(()));
}
while *this.header_read < DARE_HEADER_SIZE {
let mut read_buf = ReadBuf::new(&mut this.header_buf[*this.header_read..]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
if *this.header_read == 0 {
*this.finished = true;
return Poll::Ready(Ok(()));
}
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF while reading DARE header",
)));
}
*this.header_read += n;
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
let header = this.header_buf;
let payload_len = usize::from(u16::from_le_bytes([header[2], header[3]])) + 1;
let package_len = payload_len + DARE_TAG_SIZE;
if payload_len == 0 || payload_len > DARE_PAYLOAD_SIZE {
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "invalid DARE payload size")));
}
if !is_final_header(*header) && payload_len != DARE_PAYLOAD_SIZE {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::InvalidData,
"non-final DARE package must carry a full 64KiB payload",
)));
}
if this.ciphertext_buf.len() < package_len {
this.ciphertext_buf.resize(package_len, 0);
}
*this.ciphertext_len = package_len;
*this.ciphertext_read = 0;
while *this.ciphertext_read < *this.ciphertext_len {
let mut read_buf = ReadBuf::new(&mut this.ciphertext_buf[*this.ciphertext_read..*this.ciphertext_len]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF while reading DARE ciphertext",
)));
}
*this.ciphertext_read += n;
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
match open_dare_package(
this.cipher,
*this.sequence_number,
*this.expected_base_nonce,
*header,
&this.ciphertext_buf[..*this.ciphertext_len],
this.ref_nonce,
) {
Ok((plaintext, ref_nonce, finalized)) => {
*this.ref_nonce = Some(ref_nonce);
*this.finalized = finalized;
*this.header_read = 0;
*this.ciphertext_len = 0;
*this.ciphertext_read = 0;
*this.sequence_number = this.sequence_number.wrapping_add(1);
*this.output_buffer = plaintext;
let to_copy = min(buf.remaining(), this.output_buffer.len());
buf.put_slice(&this.output_buffer[..to_copy]);
*this.output_pos += to_copy;
if *this.output_pos == this.output_buffer.len() {
this.output_buffer.clear();
*this.output_pos = 0;
}
return Poll::Ready(Ok(()));
}
Err(err) => return Poll::Ready(Err(err)),
}
}
}
}
impl<R> EtagResolvable for DecryptReader<R>
where
R: EtagResolvable,
{
fn try_resolve_etag(&mut self) -> Option<String> {
self.inner.try_resolve_etag()
}
}
impl<R> HashReaderDetector for DecryptReader<R>
where
R: HashReaderDetector,
{
fn is_hash_reader(&self) -> bool {
self.inner.is_hash_reader()
}
fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> {
self.inner.as_hash_reader_mut()
}
}
impl<R> TryGetIndex for DecryptReader<R>
where
R: TryGetIndex,
{
fn try_get_index(&self) -> Option<&Index> {
self.inner.try_get_index()
}
}
fn build_dare_package(
cipher: &Aes256Gcm,
sequence_number: u32,
base_nonce: [u8; 12],
plaintext: &[u8],
final_package: bool,
) -> io::Result<Vec<u8>> {
let mut header = [0u8; DARE_HEADER_SIZE];
header[0] = DARE_VERSION_20;
header[1] = DARE_CIPHER_AES_256_GCM;
header[2..4].copy_from_slice(
&u16::try_from(plaintext.len() - 1)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "DARE payload too large for Version 2.0 framing"))?
.to_le_bytes(),
);
header[4..16].copy_from_slice(&base_nonce);
if final_package {
header[4] |= 0x80;
} else {
header[4] &= 0x7F;
}
let mut package_nonce = header[4..16].try_into().expect("nonce slice");
xor_sequence_into_nonce(&mut package_nonce, sequence_number);
let nonce = Nonce::try_from(package_nonce.as_slice())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid DARE nonce length"))?;
let ciphertext = cipher
.encrypt(
&nonce,
aes_gcm::aead::Payload {
msg: plaintext,
aad: &header[..4],
},
)
.map_err(|err| io::Error::other(format!("failed to encrypt DARE package: {err}")))?;
let mut package = Vec::with_capacity(DARE_HEADER_SIZE + ciphertext.len());
package.extend_from_slice(&header);
package.extend_from_slice(&ciphertext);
Ok(package)
}
fn open_dare_package(
cipher: &Aes256Gcm,
sequence_number: u32,
expected_base_nonce: Option<[u8; 12]>,
header: [u8; DARE_HEADER_SIZE],
ciphertext: &[u8],
ref_nonce: &mut Option<[u8; 12]>,
) -> io::Result<(Vec<u8>, [u8; 12], bool)> {
if header[0] != DARE_VERSION_20 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE version"));
}
if header[1] != DARE_CIPHER_AES_256_GCM {
return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE cipher suite"));
}
let header_nonce: [u8; 12] = header[4..16].try_into().expect("nonce slice");
if let Some(expected_base_nonce) = expected_base_nonce {
let masked_expected = apply_final_flag(expected_base_nonce, is_final_header(header));
if header_nonce != masked_expected {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"DARE package nonce does not match the configured stream nonce",
));
}
}
let current_ref = ref_nonce.get_or_insert(header_nonce);
let expected_ref = apply_final_flag(*current_ref, is_final_header(header));
if header_nonce != expected_ref {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"DARE package nonce does not match the stream reference nonce",
));
}
let mut package_nonce = header_nonce;
xor_sequence_into_nonce(&mut package_nonce, sequence_number);
let nonce = Nonce::try_from(package_nonce.as_slice())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid DARE nonce length"))?;
let plaintext = cipher
.decrypt(
&nonce,
aes_gcm::aead::Payload {
msg: ciphertext,
aad: &header[..4],
},
)
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("DARE authentication failed: {err}")))?;
Ok((plaintext, *current_ref, is_final_header(header)))
}
fn xor_sequence_into_nonce(nonce: &mut [u8; 12], sequence_number: u32) {
let last = u32::from_le_bytes([nonce[8], nonce[9], nonce[10], nonce[11]]) ^ sequence_number;
nonce[8..12].copy_from_slice(&last.to_le_bytes());
}
fn apply_final_flag(mut nonce: [u8; 12], final_package: bool) -> [u8; 12] {
if final_package {
nonce[0] |= 0x80;
} else {
nonce[0] &= 0x7F;
}
nonce
}
fn is_final_header(header: [u8; DARE_HEADER_SIZE]) -> bool {
header[4] & 0x80 != 0
}
pub fn derive_part_key(object_key: [u8; 32], part_number: u32) -> [u8; 32] {
let mut mac = HmacSha256::new_from_slice(&object_key).expect("HMAC-SHA256 accepts 32-byte object keys");
mac.update(&part_number.to_le_bytes());
let mut part_key = [0u8; 32];
part_key.copy_from_slice(mac.finalize().into_bytes().as_slice());
part_key
}
fn random_stream_nonce() -> [u8; 12] {
let mut nonce = [0u8; 12];
rand::rng().fill(&mut nonce);
nonce
}
#[cfg(test)]
mod tests {
use super::*;
use hex::encode as hex_encode;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
const DARE_PACKAGE_SIZE: usize = DARE_HEADER_SIZE + DARE_PAYLOAD_SIZE + DARE_TAG_SIZE;
#[tokio::test]
async fn decrypt_reader_can_start_from_non_zero_sequence_number() {
let plaintext = vec![0xAB; DARE_PAYLOAD_SIZE * 2 + 19];
let key_bytes = [0x44u8; 32];
let base_nonce = [0x66u8; 12];
let mut encrypted = Vec::new();
EncryptReader::new(Cursor::new(plaintext.clone()), key_bytes, base_nonce)
.read_to_end(&mut encrypted)
.await
.expect("encrypt plaintext");
let tail = encrypted[DARE_PACKAGE_SIZE..].to_vec();
let mut decrypted = Vec::new();
DecryptReader::new_with_sequence(Cursor::new(tail), key_bytes, base_nonce, 1)
.read_to_end(&mut decrypted)
.await
.expect("decrypt tail packages with non-zero sequence");
assert_eq!(decrypted, plaintext[DARE_PAYLOAD_SIZE..]);
}
#[tokio::test]
async fn singlepart_object_key_roundtrip_uses_header_nonce() {
let object_key = [0x51u8; 32];
let plaintext = vec![0xAC; DARE_PAYLOAD_SIZE + 17];
let mut encrypted = Vec::new();
EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key)
.read_to_end(&mut encrypted)
.await
.expect("encrypt object-key singlepart stream");
let mut decrypted = Vec::new();
DecryptReader::new_with_object_key(Cursor::new(encrypted), object_key)
.read_to_end(&mut decrypted)
.await
.expect("decrypt object-key singlepart stream");
assert_eq!(decrypted, plaintext);
}
#[test]
fn derive_part_key_matches_minio_test_vectors() {
assert_eq!(
hex_encode(derive_part_key([0u8; 32], 0)),
"aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"
);
assert_eq!(
hex_encode(derive_part_key([0u8; 32], 1)),
"a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"
);
assert_eq!(
hex_encode(derive_part_key([0u8; 32], 10_000)),
"f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"
);
}
#[tokio::test]
async fn multipart_object_key_roundtrip_resets_sequence_per_part() {
let object_key = [0x19u8; 32];
let part_one_plaintext = vec![0xA1; DARE_PAYLOAD_SIZE + 31];
let part_two_plaintext = vec![0xB2; DARE_PAYLOAD_SIZE * 2 + 7];
let mut encrypted_one = Vec::new();
EncryptReader::new_multipart_with_object_key(Cursor::new(part_one_plaintext.clone()), object_key, 1)
.read_to_end(&mut encrypted_one)
.await
.expect("encrypt multipart part one with object key");
let mut encrypted_two = Vec::new();
EncryptReader::new_multipart_with_object_key(Cursor::new(part_two_plaintext.clone()), object_key, 2)
.read_to_end(&mut encrypted_two)
.await
.expect("encrypt multipart part two with object key");
let mut encrypted = encrypted_one.clone();
encrypted.extend_from_slice(&encrypted_two);
let mut decrypted = Vec::new();
DecryptReader::new_multipart_with_object_key(Cursor::new(encrypted), object_key, vec![1, 2])
.read_to_end(&mut decrypted)
.await
.expect("decrypt multipart object-key stream");
let mut expected = part_one_plaintext;
expected.extend_from_slice(&part_two_plaintext);
assert_eq!(decrypted, expected);
assert_ne!(encrypted_one, encrypted_two[..encrypted_one.len()]);
}
}
crates/rio-v2/src/lib.rs
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! rio_v2 selectively replaces legacy rio components while keeping the
//! remaining API surface stable for feature-gated integration.
mod compress_reader;
mod encrypt_reader;
mod s2_index;
pub use compress_reader::{CompressReader, DecompressReader};
pub use encrypt_reader::{DecryptReader, EncryptReader, derive_part_key};
pub use rustfs_rio::DEFAULT_ENCRYPTION_BLOCK_SIZE;
pub use rustfs_rio::DynReader;
pub use rustfs_rio::EtagReader;
pub use rustfs_rio::EtagResolvable;
pub use rustfs_rio::HardLimitReader;
pub use rustfs_rio::HashReader;
pub use rustfs_rio::HashReaderDetector;
pub use rustfs_rio::HashReaderMut;
pub use rustfs_rio::Index;
pub use rustfs_rio::LimitReader;
pub use rustfs_rio::ReadStream;
pub use rustfs_rio::Reader;
pub use rustfs_rio::ReaderCapabilities;
pub use rustfs_rio::TryGetIndex;
pub use rustfs_rio::WarpReader;
pub use rustfs_rio::boxed_reader;
pub use rustfs_rio::read_checksums;
pub use rustfs_rio::resolve_etag_generic;
pub use rustfs_rio::wrap_reader;
pub use s2_index::{decode_minio_index_bytes, minio_index_storage_bytes};
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
const DARE_VERSION_20: u8 = 0x20;
const DARE_HEADER_SIZE: usize = 16;
const DARE_TAG_SIZE: usize = 16;
const DARE_PAYLOAD_SIZE: usize = 64 * 1024;
const DARE_PACKAGE_SIZE: usize = DARE_HEADER_SIZE + DARE_PAYLOAD_SIZE + DARE_TAG_SIZE;
#[tokio::test]
async fn encrypt_reader_emits_dare_v2_packages() {
let plaintext = vec![0x5Au8; DARE_PAYLOAD_SIZE + 17];
let key_bytes = [0x11u8; 32];
let base_nonce = [0x22u8; 12];
let mut encrypted = Vec::new();
EncryptReader::new(Cursor::new(plaintext), key_bytes, base_nonce)
.read_to_end(&mut encrypted)
.await
.expect("encrypt plaintext into rio_v2 stream");
assert!(
encrypted.len() > DARE_PACKAGE_SIZE,
"expected at least one full DARE package and one final package"
);
assert_eq!(encrypted[0], DARE_VERSION_20, "rio_v2 encrypted streams must start with a DARE V2 header");
assert_eq!(
&encrypted[4..16],
&base_nonce,
"rio_v2 should preserve the configured nonce in the first DARE header"
);
let second_header_offset = DARE_PACKAGE_SIZE;
assert_eq!(
encrypted[second_header_offset], DARE_VERSION_20,
"rio_v2 should emit subsequent DARE V2 package headers at 64KiB boundaries"
);
assert_ne!(
encrypted[second_header_offset + 4] & 0x80,
0,
"the final DARE package must set the final flag"
);
}
}
crates/rio-v2/src/s2_index.rs
+434
View File
@@ -0,0 +1,434 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use bytes::Bytes;
use rustfs_rio::Index;
use serde::Deserialize;
use std::io;
const S2_INDEX_HEADER: &[u8] = b"s2idx\x00";
const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s";
const CHUNK_TYPE_INDEX: u8 = 0x99;
const SKIPPABLE_FRAME_HEADER: usize = 4;
const MAX_INDEX_ENTRIES: usize = 1 << 16;
#[derive(Debug, Deserialize)]
struct LegacyIndexJson {
total_uncompressed: i64,
total_compressed: i64,
offsets: Vec<LegacyIndexOffset>,
est_block_uncompressed: i64,
}
#[derive(Debug, Deserialize)]
struct LegacyIndexOffset {
compressed: i64,
uncompressed: i64,
}
#[derive(Debug, Clone)]
struct S2IndexInfo {
compressed_offset: i64,
uncompressed_offset: i64,
}
#[derive(Debug, Clone)]
struct S2Index {
total_uncompressed: i64,
total_compressed: i64,
est_block_uncompressed: i64,
info: Vec<S2IndexInfo>,
}
pub fn minio_index_storage_bytes(index: &Index) -> Bytes {
let decoded = legacy_index_to_s2_index(index).unwrap_or_else(|_| S2Index {
total_uncompressed: index.total_uncompressed,
total_compressed: index.total_compressed,
est_block_uncompressed: 0,
info: Vec::new(),
});
let encoded = decoded.into_full_bytes();
remove_index_headers(encoded.as_ref())
.map(Bytes::copy_from_slice)
.unwrap_or(encoded)
}
pub fn decode_minio_index_bytes(bytes: &Bytes) -> Option<Index> {
let decoded = S2Index::load(bytes.as_ref())
.or_else(|_| S2Index::load(&restore_index_headers(bytes.as_ref())))
.ok()?;
let mut index = Index::new();
for info in decoded.info {
index.add(info.compressed_offset, info.uncompressed_offset).ok()?;
}
index.total_uncompressed = decoded.total_uncompressed;
index.total_compressed = decoded.total_compressed;
Some(index)
}
fn legacy_index_to_s2_index(index: &Index) -> io::Result<S2Index> {
let json = index
.to_json()
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
let decoded: LegacyIndexJson =
serde_json::from_slice(&json).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
Ok(S2Index {
total_uncompressed: decoded.total_uncompressed,
total_compressed: decoded.total_compressed,
est_block_uncompressed: decoded.est_block_uncompressed,
info: decoded
.offsets
.into_iter()
.map(|offset| S2IndexInfo {
compressed_offset: offset.compressed,
uncompressed_offset: offset.uncompressed,
})
.collect(),
})
}
impl S2Index {
fn into_full_bytes(self) -> Bytes {
let mut out = Vec::new();
out.extend_from_slice(&[CHUNK_TYPE_INDEX, 0, 0, 0]);
out.extend_from_slice(S2_INDEX_HEADER);
write_varint(&mut out, self.total_uncompressed);
write_varint(&mut out, self.total_compressed);
write_varint(&mut out, self.est_block_uncompressed);
write_varint(&mut out, self.info.len() as i64);
let has_uncompressed = self.has_explicit_uncompressed_offsets();
out.push(u8::from(has_uncompressed));
if has_uncompressed {
for (idx, info) in self.info.iter().enumerate() {
let mut offset = info.uncompressed_offset;
if idx > 0 {
let prev = &self.info[idx - 1];
offset -= prev.uncompressed_offset + self.est_block_uncompressed;
}
write_varint(&mut out, offset);
}
}
let mut compressed_predict = self.est_block_uncompressed / 2;
for (idx, info) in self.info.iter().enumerate() {
let mut offset = info.compressed_offset;
if idx > 0 {
let prev = &self.info[idx - 1];
offset -= prev.compressed_offset + compressed_predict;
compressed_predict += offset / 2;
}
write_varint(&mut out, offset);
}
let mut total_size = [0u8; 4];
total_size.copy_from_slice(&((out.len() + 4 + S2_INDEX_TRAILER.len()) as u32).to_le_bytes());
out.extend_from_slice(&total_size);
out.extend_from_slice(S2_INDEX_TRAILER);
let chunk_len = out.len() - SKIPPABLE_FRAME_HEADER;
out[1] = chunk_len as u8;
out[2] = (chunk_len >> 8) as u8;
out[3] = (chunk_len >> 16) as u8;
Bytes::from(out)
}
fn has_explicit_uncompressed_offsets(&self) -> bool {
for (idx, info) in self.info.iter().enumerate() {
if idx == 0 {
if info.uncompressed_offset != 0 {
return true;
}
continue;
}
if info.uncompressed_offset != self.info[idx - 1].uncompressed_offset + self.est_block_uncompressed {
return true;
}
}
false
}
fn load(mut bytes: &[u8]) -> io::Result<Self> {
if bytes.len() <= SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + S2_INDEX_TRAILER.len() {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small"));
}
if bytes[0] != CHUNK_TYPE_INDEX {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index chunk type"));
}
let chunk_len = (bytes[1] as usize) | ((bytes[2] as usize) << 8) | ((bytes[3] as usize) << 16);
bytes = &bytes[SKIPPABLE_FRAME_HEADER..];
if bytes.len() < chunk_len {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small"));
}
bytes = &bytes[..chunk_len];
if !bytes.starts_with(S2_INDEX_HEADER) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index header"));
}
bytes = &bytes[S2_INDEX_HEADER.len()..];
let (total_uncompressed, used) = read_varint(bytes)?;
if total_uncompressed < 0 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid uncompressed size"));
}
bytes = &bytes[used..];
let (total_compressed, used) = read_varint(bytes)?;
bytes = &bytes[used..];
let (est_block_uncompressed, used) = read_varint(bytes)?;
if est_block_uncompressed < 0 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid block size"));
}
bytes = &bytes[used..];
let (entries, used) = read_varint(bytes)?;
if entries < 0 || entries > MAX_INDEX_ENTRIES as i64 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid number of entries"));
}
bytes = &bytes[used..];
if bytes.is_empty() {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small"));
}
let has_uncompressed = bytes[0];
if has_uncompressed & 1 != has_uncompressed {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid uncompressed flag"));
}
bytes = &bytes[1..];
let mut info = vec![
S2IndexInfo {
compressed_offset: 0,
uncompressed_offset: 0,
};
entries as usize
];
for idx in 0..info.len() {
let mut uncompressed_offset = 0_i64;
if has_uncompressed != 0 {
let (value, used) = read_varint(bytes)?;
uncompressed_offset = value;
bytes = &bytes[used..];
}
if idx > 0 {
let prev = info[idx - 1].uncompressed_offset;
uncompressed_offset += prev + est_block_uncompressed;
if uncompressed_offset <= prev {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid uncompressed offset"));
}
}
if uncompressed_offset < 0 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "negative uncompressed offset"));
}
info[idx].uncompressed_offset = uncompressed_offset;
}
let mut compressed_predict = est_block_uncompressed / 2;
for idx in 0..info.len() {
let (mut compressed_offset, used) = read_varint(bytes)?;
bytes = &bytes[used..];
if idx > 0 {
let next_compressed_predict = compressed_predict + compressed_offset / 2;
let prev = info[idx - 1].compressed_offset;
compressed_offset += prev + compressed_predict;
if compressed_offset <= prev {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid compressed offset"));
}
compressed_predict = next_compressed_predict;
}
if compressed_offset < 0 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "negative compressed offset"));
}
info[idx].compressed_offset = compressed_offset;
}
if bytes.len() < 4 + S2_INDEX_TRAILER.len() {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small"));
}
bytes = &bytes[4..];
if !bytes.starts_with(S2_INDEX_TRAILER) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index trailer"));
}
Ok(Self {
total_uncompressed,
total_compressed,
est_block_uncompressed,
info,
})
}
}
fn remove_index_headers(bytes: &[u8]) -> Option<&[u8]> {
let save = SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + S2_INDEX_TRAILER.len() + 4;
if bytes.len() <= save || bytes[0] != CHUNK_TYPE_INDEX {
return None;
}
let chunk_len = (bytes[1] as usize) | ((bytes[2] as usize) << 8) | ((bytes[3] as usize) << 16);
let bytes = &bytes[SKIPPABLE_FRAME_HEADER..];
if bytes.len() < chunk_len {
return None;
}
let bytes = &bytes[..chunk_len];
let bytes = bytes.strip_prefix(S2_INDEX_HEADER)?;
let bytes = bytes.strip_suffix(S2_INDEX_TRAILER)?;
if bytes.len() < 4 {
return None;
}
Some(&bytes[..bytes.len() - 4])
}
fn restore_index_headers(input: &[u8]) -> Vec<u8> {
if input.is_empty() {
return Vec::new();
}
let mut bytes = Vec::with_capacity(SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + input.len() + 4 + S2_INDEX_TRAILER.len());
bytes.extend_from_slice(&[CHUNK_TYPE_INDEX, 0, 0, 0]);
bytes.extend_from_slice(S2_INDEX_HEADER);
bytes.extend_from_slice(input);
bytes.extend_from_slice(&((bytes.len() + 4 + S2_INDEX_TRAILER.len()) as u32).to_le_bytes());
bytes.extend_from_slice(S2_INDEX_TRAILER);
let chunk_len = bytes.len() - SKIPPABLE_FRAME_HEADER;
bytes[1] = chunk_len as u8;
bytes[2] = (chunk_len >> 8) as u8;
bytes[3] = (chunk_len >> 16) as u8;
bytes
}
fn write_varint(out: &mut Vec<u8>, value: i64) {
let mut unsigned = ((value as u64) << 1) ^ ((value >> 63) as u64);
while unsigned >= 0x80 {
out.push((unsigned as u8) | 0x80);
unsigned >>= 7;
}
out.push(unsigned as u8);
}
fn read_varint(bytes: &[u8]) -> io::Result<(i64, usize)> {
let (unsigned, used) = read_uvarint(bytes)?;
let value = ((unsigned >> 1) as i64) ^ (-((unsigned & 1) as i64));
Ok((value, used))
}
fn read_uvarint(bytes: &[u8]) -> io::Result<(u64, usize)> {
let mut value = 0_u64;
let mut shift = 0_u32;
for (idx, byte) in bytes.iter().copied().enumerate() {
if byte < 0x80 {
if idx > 9 || (idx == 9 && byte > 1) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "varint overflow"));
}
return Ok((value | ((byte as u64) << shift), idx + 1));
}
value |= ((byte & 0x7f) as u64) << shift;
shift += 7;
}
Err(io::Error::new(io::ErrorKind::UnexpectedEof, "unexpected EOF while reading varint"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn signed_varint_matches_go_binary_varint_examples() {
let cases = [
(0, vec![0x00]),
(-1, vec![0x01]),
(1, vec![0x02]),
(-2, vec![0x03]),
(64, vec![0x80, 0x01]),
(-64, vec![0x7f]),
];
for (value, expected) in cases {
let mut encoded = Vec::new();
write_varint(&mut encoded, value);
assert_eq!(encoded, expected);
assert_eq!(read_varint(&encoded).unwrap(), (value, encoded.len()));
}
}
#[test]
fn minio_storage_bytes_round_trip_through_headerless_form() {
let mut source = Index::new();
source.add(0, 0).unwrap();
source.add(1_048_576, 2_097_152).unwrap();
source.total_uncompressed = 4_194_304;
source.total_compressed = 3_000_000;
let stored = minio_index_storage_bytes(&source);
assert!(!stored.starts_with(&[CHUNK_TYPE_INDEX, 0x2a, 0x4d, 0x18]));
assert_eq!(
S2Index::load(&restore_index_headers(&stored))
.expect("restore full index")
.info
.len(),
2
);
let decoded = decode_minio_index_bytes(&stored).expect("decode headerless MinIO index");
assert_eq!(decoded.total_uncompressed, source.total_uncompressed);
assert_eq!(decoded.total_compressed, source.total_compressed);
assert_eq!(decoded.find(2_097_152).unwrap(), (1_048_576, 2_097_152));
}
#[test]
fn minio_index_allows_unknown_total_compressed_size() {
let index = S2Index {
total_uncompressed: 4_194_304,
total_compressed: -1,
est_block_uncompressed: 0,
info: vec![
S2IndexInfo {
compressed_offset: 0,
uncompressed_offset: 0,
},
S2IndexInfo {
compressed_offset: 1_048_576,
uncompressed_offset: 2_097_152,
},
],
};
let full = index.into_full_bytes();
assert_eq!(full[0], CHUNK_TYPE_INDEX);
let headerless = Bytes::copy_from_slice(remove_index_headers(full.as_ref()).expect("strip index headers"));
let restored = restore_index_headers(&headerless);
assert_eq!(restored[0], CHUNK_TYPE_INDEX);
let decoded = decode_minio_index_bytes(&headerless).expect("decode index with unknown compressed total");
assert_eq!(decoded.total_uncompressed, 4_194_304);
assert_eq!(decoded.total_compressed, -1);
assert_eq!(decoded.find(2_097_152).unwrap(), (1_048_576, 2_097_152));
}
}
crates/rio-v2/tests/README.md
+55
View File
@@ -0,0 +1,55 @@
# rio-v2 Fixture Tests
This directory holds interoperability tests for `rustfs-rio-v2`.
## Generated MinIO fixtures
The integration test `minio_generated_fixtures.rs` reads raw MinIO backend data generated by
`crates/rio-v2/tests/minio_fixture_lab/lab.py`.
Default fixture root:
```text
crates/rio-v2/tests/fixtures/minio-generated
```
The generated MinIO fixture data is intentionally not checked in and these tests are
ignored by default, so they are not part of automated CI runs.
To generate or expand fixtures locally, use:
```powershell
uv run python .\minio_fixture_lab\lab.py capture-matrix `
--root .\rustfs\crates\rio-v2\tests\fixtures\minio-generated `
--minio-binary .\rustfs\tmp\minio.darwin-arm64.RELEASE.2025-09-07T16-13-09Z `
--endpoint https://127.0.0.1:19000 `
--disk-count 1
```
Or point the tests at another generated root:
```powershell
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-smoke'
cargo +1.95.0 test -p rustfs-rio-v2 --test minio_generated_fixtures -- --ignored
```
## Scope
The current fixture test covers the full default capture matrix:
- `sse-s3-singlepart-64k`
- `sse-s3-multipart-8m`
- `sse-kms-singlepart-64k`
- `sse-kms-multipart-8m`
- `sse-c-singlepart-64k`
- `sse-c-multipart-8m`
It checks that:
- raw `xl.meta` can be parsed through `rustfs-filemeta`
- singlepart and multipart fixture structures are recognized
- SSE-S3, SSE-KMS, and SSE-C metadata markers survive capture
- KMS key ids are derived from each fixture's own `manifest.json`, so local static-KMS runs are not tied to one hard-coded key name
- SSE-C `HEAD` responses round-trip the expected customer algorithm and customer-key MD5
These tests do not yet validate full plaintext reconstruction from MinIO-written encrypted data.
crates/rio-v2/tests/fixtures/.gitignore
+2
View File
@@ -0,0 +1,2 @@
*
!.gitignore
crates/rio-v2/tests/minio_fixture_lab/README.md
+150
View File
@@ -0,0 +1,150 @@
# MinIO Fixture Lab
This lab captures real MinIO backend artifacts into a repeatable local layout for RustFS compatibility tests.
It now supports two workflows:
- manual capture from an already exported backend tree
- automated capture from a disposable local MinIO run
## Scope
Use the manual path after you already have:
- a running MinIO instance
- one or more uploaded objects you want to preserve as fixtures
- the backend object tree or object-version directory you want to export
Use the automated path when you want the lab to:
- locate a local `minio` binary
- start a disposable MinIO instance
- upload a predefined SSE fixture case
- export the generated backend tree into the lab layout
## Layout
The default root is `artifacts/minio-fixture-lab`, which is already ignored by the repository.
Each case is stored under:
```text
artifacts/minio-fixture-lab/
cases/
<case-id>/
backend/
request.json
head.json
plaintext.sha256
manifest.json
```
`manifest.json` is the source of truth for the captured case.
## Commands
Initialize the lab root:
```powershell
uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py init
```
Capture one case from an existing MinIO backend tree:
```powershell
uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py add-case `
--case-id sse-kms-singlepart-64k `
--bucket demo `
--object dir/object.bin `
--source-tree D:\minio-data-export\case-tree `
--head-json D:\minio-data-export\head.json `
--request-json D:\minio-data-export\request.json `
--plaintext-sha256 D:\minio-data-export\plaintext.sha256
```
Capture the default automated matrix:
```powershell
uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py capture-matrix `
--root D:\Github\rustfs\artifacts\minio-fixture-lab `
--minio-binary D:\go\bin\minio.exe `
--endpoint https://127.0.0.1:19000
```
Capture only one automated case:
```powershell
uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py capture-matrix `
--root D:\Github\rustfs\artifacts\minio-fixture-lab `
--minio-binary D:\Github\rustfs\tmp\minio.windows-amd64.RELEASE.2025-09-07T16-13-09Z.exe `
--endpoint https://127.0.0.1:19000 `
--case-id sse-s3-singlepart-64k
```
The automated default matrix is intentionally small:
- `sse-s3-singlepart-64k`
- `sse-kms-singlepart-64k`
- `sse-c-singlepart-64k`
- `sse-s3-multipart-8m`
- `sse-kms-multipart-8m`
- `sse-c-multipart-8m`
`64 KiB multipart` is intentionally excluded because S3 multipart semantics require a larger non-final part size.
## Automated Runner Prerequisites
The automated runner expects:
- either `minio` in `PATH`, the bundled `D:\Github\rustfs\tmp\minio.windows-amd64.RELEASE.2025-09-07T16-13-09Z.exe`, `--minio-binary`, or `--minio-root` pointing at a directory containing `minio.exe`
- a free local endpoint port
When the selected matrix includes SSE-C cases, use an `https://` endpoint. The lab will mint a short-lived local self-signed certificate and use its built-in SigV4 S3 client with certificate verification disabled for the disposable local MinIO run.
The runner provisions backend directories under `--work-root` and exports the resulting backend tree into the lab.
For local static-KMS runs, pass `--kms-secret-key` or set
`MINIO_FIXTURE_LAB_KMS_SECRET_KEY` using MinIO's
`<key-id>:<base64-32byte-key>` format. The runner derives the SSE-KMS request
key id from that configured key name automatically.
On some Windows MinIO builds, a multi-disk backend may not come online when all disk directories live on the same volume. If you only want a local smoke run of the upload/export pipeline, try:
```powershell
uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py capture-matrix `
--root D:\Github\rustfs\artifacts\minio-fixture-lab `
--minio-binary D:\go\bin\minio.exe `
--endpoint https://127.0.0.1:19000 `
--disk-count 1 `
--case-id sse-s3-singlepart-64k
```
That is a runner smoke path only. Real compatibility fixtures should still prefer the intended multi-disk backend layout when the local environment can support it.
## Capture Guidance
For each case, preserve these inputs when possible:
- the exact backend tree that contains `xl.meta` and part files
- the request shape used to create the object
- the API metadata returned by `HEAD Object`
- the plaintext digest used for byte-for-byte read verification
Recommended early matrix:
- singlepart SSE-S3
- singlepart SSE-KMS
- multipart SSE-S3
- multipart SSE-KMS
- compressed + encrypted
- range-sensitive sizes around `64 KiB` and `8 MiB`
## Next Stage
The current runner covers launch, upload, HEAD capture, and backend export.
The next iteration should focus on:
- proving the multi-disk backend path in the target local environment
- adding compressed fixtures to the automated matrix
- tightening exported tree selection if a narrower object-level slice becomes practical
crates/rio-v2/tests/minio_fixture_lab/lab.py
+963
View File
@@ -0,0 +1,963 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import base64
import hashlib
import hmac
import ipaddress
import json
import os
import shutil
import ssl
import subprocess
import sys
import time
import urllib.parse
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
DEFAULT_ROOT = Path("artifacts/minio-fixture-lab")
DEFAULT_WORK_ROOT = DEFAULT_ROOT / "_runner"
DEFAULT_MINIO_BINARY = (
Path(__file__).resolve().parents[4]
/ "tmp"
/ "minio.darwin-arm64.RELEASE.2025-09-07T16-13-09Z"
)
DEFAULT_KMS_KEY_ID = "minio-default-key"
DEFAULT_KMS_SECRET_KEY = (
"minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g="
)
LAB_KMS_SECRET_KEY_ENV = "MINIO_FIXTURE_LAB_KMS_SECRET_KEY"
DEFAULT_ENDPOINT = "http://127.0.0.1:9000"
DEFAULT_DISK_COUNT = 4
DEFAULT_ACCESS_KEY = "minioadmin"
DEFAULT_SECRET_KEY = "minioadmin"
DEFAULT_REGION = "us-east-1"
MULTIPART_CHUNK_SIZE = 5 * 1024 * 1024
DEFAULT_BUCKET = "demo"
DEFAULT_OBJECT = "dir/object.bin"
DEFAULT_SSE_C_KEY_BYTES = bytes(range(32))
DEFAULT_SSE_C_KEY_B64 = base64.b64encode(DEFAULT_SSE_C_KEY_BYTES).decode("ascii")
DEFAULT_SSE_C_KEY_MD5_B64 = base64.b64encode(hashlib.md5(DEFAULT_SSE_C_KEY_BYTES).digest()).decode("ascii")
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
def ensure_dir(path: Path) -> Path:
path.mkdir(parents=True, exist_ok=True)
return path
def copy_optional_file(source: Path | None, destination: Path) -> str | None:
if source is None:
return None
if not source.is_file():
raise FileNotFoundError(f"expected file: {source}")
shutil.copy2(source, destination)
return destination.name
def copy_tree(source: Path, destination: Path) -> list[str]:
if not source.exists():
raise FileNotFoundError(f"source tree does not exist: {source}")
if source.is_file():
raise ValueError(f"source tree must be a directory: {source}")
if destination.exists():
shutil.rmtree(destination)
shutil.copytree(source, destination)
return sorted(
str(path.relative_to(destination)).replace("\\", "/")
for path in destination.rglob("*")
if path.is_file()
)
def write_json(path: Path, payload: dict[str, Any]) -> None:
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def read_optional_json(path: Path | None) -> dict[str, Any] | None:
if path is None:
return None
return json.loads(path.read_text(encoding="utf-8"))
def format_context_header(context: dict[str, str] | None) -> str | None:
if not context:
return None
json_bytes = json.dumps(context, sort_keys=True, separators=(",", ":")).encode("utf-8")
return base64.b64encode(json_bytes).decode("ascii")
def format_size_label(size_bytes: int) -> str:
if size_bytes == 64 * 1024:
return "64k"
if size_bytes == 8 * 1024 * 1024:
return "8m"
return f"{size_bytes}b"
@dataclass
class LabPaths:
root: Path
cases: Path
@dataclass
class MinioLauncher:
kind: str
command: list[str]
workdir: Path
@dataclass
class FixtureCase:
case_id: str
bucket: str
object_name: str
encryption: str
size_bytes: int
multipart: bool
kms_key_id: str | None = None
kms_context: dict[str, str] | None = None
@dataclass(frozen=True)
class KmsSecretKeyConfig:
secret_key: str
key_id: str
def resolve_lab_paths(root: Path) -> LabPaths:
root = root.resolve()
return LabPaths(root=root, cases=root / "cases")
def resolve_existing_command(name: str) -> Path:
resolved = shutil.which(name)
if resolved is None:
raise FileNotFoundError(f"required command not found in PATH: {name}")
return Path(resolved).resolve()
def discover_minio_launcher(
explicit_binary: Path | None, minio_root: Path | None
) -> MinioLauncher:
if explicit_binary is not None:
binary = explicit_binary.resolve()
if not binary.is_file():
raise FileNotFoundError(f"minio binary not found: {binary}")
return MinioLauncher(kind="binary", command=[str(binary)], workdir=binary.parent)
resolved_root = minio_root.resolve() if minio_root is not None else None
if resolved_root is not None:
candidate_binary = resolved_root / "minio.exe"
if candidate_binary.is_file():
return MinioLauncher(
kind="binary",
command=[str(candidate_binary.resolve())],
workdir=candidate_binary.resolve().parent,
)
if DEFAULT_MINIO_BINARY.is_file():
binary = DEFAULT_MINIO_BINARY.resolve()
return MinioLauncher(kind="binary", command=[str(binary)], workdir=binary.parent)
path_binary = shutil.which("minio")
if path_binary is not None:
binary = Path(path_binary).resolve()
return MinioLauncher(kind="binary", command=[str(binary)], workdir=binary.parent)
raise FileNotFoundError(
"unable to locate MinIO. Provide --minio-binary, point --minio-root at a directory "
"containing minio.exe, or ensure minio is in PATH."
)
def parse_kms_secret_key(secret_key: str) -> KmsSecretKeyConfig:
key_id, separator, raw_key = secret_key.partition(":")
if not separator or not key_id or not raw_key:
raise ValueError(
"kms secret key must be in the form '<key-id>:<base64-32byte-key>'"
)
return KmsSecretKeyConfig(secret_key=secret_key, key_id=key_id)
def resolve_kms_secret_key(explicit_secret_key: str | None) -> KmsSecretKeyConfig:
secret_key = (
explicit_secret_key
or os.environ.get(LAB_KMS_SECRET_KEY_ENV)
or os.environ.get("MINIO_KMS_SECRET_KEY")
or DEFAULT_KMS_SECRET_KEY
)
return parse_kms_secret_key(secret_key)
def build_default_cases(kms_key_id: str = DEFAULT_KMS_KEY_ID) -> list[FixtureCase]:
kms_context = {"project": "rio-v2", "stage": "fixture"}
cases = []
for encryption in ("SSE-S3", "SSE-KMS", "SSE-C"):
for multipart, size_bytes in ((False, 64 * 1024), (True, 8 * 1024 * 1024)):
shape = "multipart" if multipart else "singlepart"
case = FixtureCase(
case_id=f"{encryption.lower()}-{shape}-{format_size_label(size_bytes)}",
bucket=DEFAULT_BUCKET,
object_name=DEFAULT_OBJECT,
encryption=encryption,
size_bytes=size_bytes,
multipart=multipart,
kms_key_id=kms_key_id if encryption == "SSE-KMS" else None,
kms_context=kms_context if encryption == "SSE-KMS" else None,
)
cases.append(case)
return cases
def build_request_record(case: FixtureCase) -> dict[str, Any]:
headers: dict[str, str] = {}
if case.encryption == "SSE-S3":
headers["x-amz-server-side-encryption"] = "AES256"
elif case.encryption == "SSE-KMS":
headers["x-amz-server-side-encryption"] = "aws:kms"
if case.kms_key_id is not None:
headers["x-amz-server-side-encryption-aws-kms-key-id"] = case.kms_key_id
context_header = format_context_header(case.kms_context)
if context_header is not None:
headers["x-amz-server-side-encryption-context"] = context_header
elif case.encryption == "SSE-C":
headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
headers["x-amz-server-side-encryption-customer-key"] = DEFAULT_SSE_C_KEY_B64
headers["x-amz-server-side-encryption-customer-key-md5"] = DEFAULT_SSE_C_KEY_MD5_B64
else:
raise ValueError(f"unsupported encryption mode: {case.encryption}")
record: dict[str, Any] = {
"case_id": case.case_id,
"bucket": case.bucket,
"object": case.object_name,
"encryption": case.encryption,
"multipart": case.multipart,
"size_bytes": case.size_bytes,
"headers": headers,
}
if case.multipart:
record["multipart_chunk_size_bytes"] = MULTIPART_CHUNK_SIZE
return record
def cmd_init(args: argparse.Namespace) -> int:
paths = resolve_lab_paths(args.root)
ensure_layout(paths)
print(str(paths.root))
return 0
def ensure_layout(paths: LabPaths) -> None:
ensure_dir(paths.root)
ensure_dir(paths.cases)
write_json(
paths.root / "layout.json",
{
"schema_version": 1,
"created_at_utc": utc_now(),
"root": ".",
"description": "Captured MinIO fixture cases for RustFS compatibility validation.",
},
)
def store_case_artifacts(
*,
paths: LabPaths,
case_id: str,
bucket: str,
object_name: str,
source_tree: Path,
version_id: str | None,
request_payload: dict[str, Any] | None,
head_payload: dict[str, Any] | None,
plaintext_sha256: str | None,
notes: str | None,
capture_payload: dict[str, Any] | None = None,
) -> Path:
ensure_dir(paths.cases)
case_dir = paths.cases / case_id
backend_dir = case_dir / "backend"
if case_dir.exists():
shutil.rmtree(case_dir)
ensure_dir(case_dir)
exported_files = copy_tree(source_tree.resolve(), backend_dir)
request_file = None
if request_payload is not None:
write_json(case_dir / "request.json", request_payload)
request_file = "request.json"
head_file = None
if head_payload is not None:
write_json(case_dir / "head.json", head_payload)
head_file = "head.json"
plaintext_file = None
if plaintext_sha256 is not None:
(case_dir / "plaintext.sha256").write_text(plaintext_sha256 + "\n", encoding="utf-8")
plaintext_file = "plaintext.sha256"
manifest = {
"schema_version": 1,
"case_id": case_id,
"created_at_utc": utc_now(),
"bucket": bucket,
"object": object_name,
"version_id": version_id,
"source_tree": str(source_tree).replace("\\", "/"),
"notes": notes,
"artifacts": {
"backend_dir": "backend",
"request_json": request_file,
"head_json": head_file,
"plaintext_sha256": plaintext_file,
},
"backend_files": exported_files,
}
if capture_payload is not None:
manifest["capture"] = capture_payload
write_json(case_dir / "manifest.json", manifest)
return case_dir
def cmd_add_case(args: argparse.Namespace) -> int:
paths = resolve_lab_paths(args.root)
case_dir = store_case_artifacts(
paths=paths,
case_id=args.case_id,
bucket=args.bucket,
object_name=args.object,
source_tree=args.source_tree,
version_id=args.version_id,
request_payload=read_optional_json(args.request_json.resolve() if args.request_json else None),
head_payload=read_optional_json(args.head_json.resolve() if args.head_json else None),
plaintext_sha256=(
args.plaintext_sha256.resolve().read_text(encoding="utf-8").strip()
if args.plaintext_sha256
else None
),
notes=args.notes,
)
print(str(case_dir))
return 0
def parse_endpoint(endpoint: str) -> tuple[str, int]:
parsed = urlparse(endpoint)
if parsed.scheme not in {"http", "https"}:
raise ValueError(f"unsupported endpoint scheme: {endpoint}")
if parsed.hostname is None or parsed.port is None:
raise ValueError(f"endpoint must include host and port: {endpoint}")
return parsed.hostname, parsed.port
def is_https_endpoint(endpoint: str) -> bool:
return urlparse(endpoint).scheme == "https"
def ensure_local_tls_certificates(certs_dir: Path, hostname: str) -> None:
public_crt = certs_dir / "public.crt"
private_key = certs_dir / "private.key"
if public_crt.is_file() and private_key.is_file():
return
ensure_dir(certs_dir)
openssl = resolve_existing_command("openssl")
san_entries = ["DNS:localhost", "IP:127.0.0.1"]
try:
ipaddress.ip_address(hostname)
san_entries.append(f"IP:{hostname}")
except ValueError:
san_entries.append(f"DNS:{hostname}")
config_path = certs_dir / "openssl.cnf"
config_path.write_text(
"\n".join(
[
"[req]",
"distinguished_name = req_distinguished_name",
"x509_extensions = v3_req",
"prompt = no",
"[req_distinguished_name]",
f"CN = {hostname}",
"[v3_req]",
f"subjectAltName = {','.join(san_entries)}",
"",
]
),
encoding="utf-8",
)
subprocess.run(
[
str(openssl),
"req",
"-x509",
"-newkey",
"rsa:2048",
"-nodes",
"-days",
"30",
"-keyout",
str(private_key),
"-out",
str(public_crt),
"-config",
str(config_path),
"-extensions",
"v3_req",
],
check=True,
capture_output=True,
text=True,
)
def build_server_command(
launcher: MinioLauncher, endpoint: str, disk_paths: list[Path], certs_dir: Path | None = None
) -> list[str]:
host, port = parse_endpoint(endpoint)
normalized_disk_paths = [str(path.resolve()).replace("\\", "/") for path in disk_paths]
command = launcher.command + [
"server",
"--address",
f"{host}:{port}",
"--console-address",
f"{host}:{port + 1}",
]
if certs_dir is not None:
command += ["--certs-dir", str(certs_dir.resolve())]
command += normalized_disk_paths
return command
def create_disk_paths(workdir: Path, disk_count: int) -> list[Path]:
disks = []
for index in range(1, disk_count + 1):
disk = ensure_dir(workdir / "backend" / f"disk{index}")
disks.append(disk)
return disks
def build_payload_file(path: Path, size_bytes: int) -> str:
hasher = hashlib.sha256()
remaining = size_bytes
pattern = bytes(range(251))
with path.open("wb") as handle:
while remaining > 0:
chunk = pattern[: min(len(pattern), remaining)]
handle.write(chunk)
hasher.update(chunk)
remaining -= len(chunk)
return hasher.hexdigest()
def minio_env(kms_secret_key: KmsSecretKeyConfig) -> dict[str, str]:
env = dict(os.environ)
env["CI"] = "on"
env["MINIO_CI_CD"] = "1"
env["MINIO_ROOT_USER"] = "minioadmin"
env["MINIO_ROOT_PASSWORD"] = "minioadmin"
env["MINIO_KMS_SECRET_KEY"] = kms_secret_key.secret_key
env["MINIO_UPDATE"] = "off"
return env
class S3Client:
def __init__(
self,
endpoint: str,
access_key: str = DEFAULT_ACCESS_KEY,
secret_key: str = DEFAULT_SECRET_KEY,
region: str = DEFAULT_REGION,
) -> None:
self.endpoint = endpoint.rstrip("/")
self.access_key = access_key
self.secret_key = secret_key
self.region = region
self.parsed_endpoint = urlparse(self.endpoint)
self.ssl_context = ssl._create_unverified_context() if is_https_endpoint(endpoint) else None
def request(
self,
method: str,
bucket: str | None = None,
key: str | None = None,
*,
query: dict[str, str] | None = None,
headers: dict[str, str] | None = None,
body: bytes = b"",
) -> tuple[int, dict[str, str], bytes]:
path = "/"
if bucket is not None:
path += urllib.parse.quote(bucket, safe="")
if key is not None:
path += "/" + urllib.parse.quote(key, safe="/")
query = query or {}
headers = dict(headers or {})
signed_headers = self.sign_headers(method, path, query, headers, body)
url = self.endpoint + path
if query:
url += "?" + urllib.parse.urlencode(sorted(query.items()), quote_via=urllib.parse.quote)
request = urllib.request.Request(url, data=body if method != "HEAD" else None, method=method)
for name, value in signed_headers.items():
request.add_header(name, value)
try:
with urllib.request.urlopen(request, timeout=30, context=self.ssl_context) as response:
return response.status, {k.lower(): v for k, v in response.headers.items()}, response.read()
except urllib.error.HTTPError as err:
payload = err.read()
raise RuntimeError(
f"S3 request failed: {method} {url}\nstatus: {err.code}\nbody:\n{payload.decode('utf-8', errors='replace')}"
) from err
def sign_headers(
self,
method: str,
canonical_uri: str,
query: dict[str, str],
headers: dict[str, str],
body: bytes,
) -> dict[str, str]:
now = datetime.now(timezone.utc)
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
date_stamp = now.strftime("%Y%m%d")
payload_hash = hashlib.sha256(body).hexdigest()
host = self.parsed_endpoint.netloc
signed_headers = {k.lower(): v.strip() for k, v in headers.items()}
signed_headers["host"] = host
signed_headers["x-amz-content-sha256"] = payload_hash
signed_headers["x-amz-date"] = amz_date
canonical_query = "&".join(
f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(v, safe='-_.~')}"
for k, v in sorted(query.items())
)
canonical_headers = "".join(f"{k}:{signed_headers[k]}\n" for k in sorted(signed_headers))
signed_header_names = ";".join(sorted(signed_headers))
canonical_request = "\n".join(
[
method,
canonical_uri,
canonical_query,
canonical_headers,
signed_header_names,
payload_hash,
]
)
scope = f"{date_stamp}/{self.region}/s3/aws4_request"
string_to_sign = "\n".join(
[
"AWS4-HMAC-SHA256",
amz_date,
scope,
hashlib.sha256(canonical_request.encode("utf-8")).hexdigest(),
]
)
signature = hmac.new(
self.signing_key(date_stamp),
string_to_sign.encode("utf-8"),
hashlib.sha256,
).hexdigest()
signed_headers["authorization"] = (
f"AWS4-HMAC-SHA256 Credential={self.access_key}/{scope}, "
f"SignedHeaders={signed_header_names}, Signature={signature}"
)
return signed_headers
def signing_key(self, date_stamp: str) -> bytes:
key = ("AWS4" + self.secret_key).encode("utf-8")
for value in (date_stamp, self.region, "s3", "aws4_request"):
key = hmac.new(key, value.encode("utf-8"), hashlib.sha256).digest()
return key
def list_buckets(self) -> None:
self.request("GET")
def create_bucket(self, bucket: str) -> None:
self.request("PUT", bucket=bucket)
def put_object(
self,
bucket: str,
key: str,
body: bytes,
headers: dict[str, str],
) -> None:
self.request("PUT", bucket=bucket, key=key, headers=headers, body=body)
def create_multipart_upload(self, bucket: str, key: str, headers: dict[str, str]) -> str:
_, _, body = self.request("POST", bucket=bucket, key=key, query={"uploads": ""}, headers=headers)
root = ET.fromstring(body)
upload_id = root.findtext("{*}UploadId")
if upload_id is None:
raise RuntimeError(f"missing UploadId in create multipart response: {body!r}")
return upload_id
def upload_part(
self,
bucket: str,
key: str,
upload_id: str,
part_number: int,
body: bytes,
headers: dict[str, str],
) -> str:
_, response_headers, _ = self.request(
"PUT",
bucket=bucket,
key=key,
query={"partNumber": str(part_number), "uploadId": upload_id},
headers=headers,
body=body,
)
etag = response_headers.get("etag")
if etag is None:
raise RuntimeError("missing ETag in upload part response")
return etag
def complete_multipart_upload(
self,
bucket: str,
key: str,
upload_id: str,
parts: list[dict[str, Any]],
) -> None:
body = build_complete_multipart_xml(parts)
self.request(
"POST",
bucket=bucket,
key=key,
query={"uploadId": upload_id},
headers={"content-type": "application/xml"},
body=body,
)
def head_object(self, bucket: str, key: str, headers: dict[str, str]) -> dict[str, Any]:
_, response_headers, _ = self.request("HEAD", bucket=bucket, key=key, headers=headers)
return {
"ContentLength": int(response_headers.get("content-length", "0")),
"ServerSideEncryption": response_headers.get("x-amz-server-side-encryption"),
"SSECustomerAlgorithm": response_headers.get("x-amz-server-side-encryption-customer-algorithm"),
"SSECustomerKeyMD5": response_headers.get("x-amz-server-side-encryption-customer-key-md5"),
"SSEKMSKeyId": response_headers.get("x-amz-server-side-encryption-aws-kms-key-id"),
"VersionId": response_headers.get("x-amz-version-id"),
}
def wait_for_minio(endpoint: str, process: subprocess.Popen[str], timeout_seconds: int) -> None:
deadline = time.time() + timeout_seconds
health_url = endpoint.rstrip("/") + "/minio/health/live"
ssl_context = ssl._create_unverified_context() if is_https_endpoint(endpoint) else None
while time.time() < deadline:
if process.poll() is not None:
raise RuntimeError("MinIO exited before becoming healthy")
try:
with urllib.request.urlopen(health_url, timeout=2, context=ssl_context) as response:
if response.status == 200:
return
except (urllib.error.URLError, TimeoutError):
time.sleep(1)
raise TimeoutError(f"timed out waiting for MinIO health endpoint: {health_url}")
def wait_for_s3_ready(client: S3Client, timeout_seconds: int) -> None:
deadline = time.time() + timeout_seconds
while time.time() < deadline:
try:
client.list_buckets()
return
except RuntimeError:
time.sleep(1)
raise TimeoutError(f"timed out waiting for S3 API readiness: {client.endpoint}")
def create_bucket(client: S3Client, case: FixtureCase) -> None:
client.create_bucket(case.bucket)
def build_complete_multipart_xml(parts: list[dict[str, Any]]) -> bytes:
root = ET.Element("CompleteMultipartUpload")
for part in parts:
part_element = ET.SubElement(root, "Part")
ET.SubElement(part_element, "PartNumber").text = str(part["PartNumber"])
ET.SubElement(part_element, "ETag").text = part["ETag"]
return ET.tostring(root, encoding="utf-8", xml_declaration=True)
def upload_case(client: S3Client, case: FixtureCase, payload_file: Path) -> dict[str, Any]:
request_record = build_request_record(case)
headers = request_record["headers"]
headers["content-type"] = "binary/octet-stream"
if not case.multipart:
client.put_object(case.bucket, case.object_name, payload_file.read_bytes(), headers)
return request_record
upload_id = client.create_multipart_upload(case.bucket, case.object_name, headers)
parts = []
with payload_file.open("rb") as handle:
part_number = 1
while True:
chunk = handle.read(MULTIPART_CHUNK_SIZE)
if not chunk:
break
part_file = payload_file.parent / f"part-{part_number}.bin"
part_file.write_bytes(chunk)
part_headers = headers if case.encryption == "SSE-C" else {}
etag = client.upload_part(
case.bucket,
case.object_name,
upload_id,
part_number,
chunk,
part_headers,
)
parts.append({"ETag": etag, "PartNumber": part_number})
part_number += 1
client.complete_multipart_upload(case.bucket, case.object_name, upload_id, parts)
return request_record
def head_case(client: S3Client, case: FixtureCase) -> dict[str, Any]:
headers = build_request_record(case)["headers"] if case.encryption == "SSE-C" else {}
return client.head_object(case.bucket, case.object_name, headers)
def stop_process(process: subprocess.Popen[str]) -> None:
if process.poll() is not None:
return
process.terminate()
try:
process.wait(timeout=15)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=15)
def capture_case(
*,
paths: LabPaths,
launcher: MinioLauncher,
endpoint: str,
case: FixtureCase,
work_root: Path,
disk_count: int,
timeout_seconds: int,
preserve_workdir: bool,
kms_secret_key: KmsSecretKeyConfig,
) -> Path:
case_workdir = work_root / case.case_id
if case_workdir.exists():
shutil.rmtree(case_workdir)
ensure_dir(case_workdir)
payload_file = case_workdir / "payload.bin"
plaintext_sha256 = build_payload_file(payload_file, case.size_bytes)
disk_paths = create_disk_paths(case_workdir, disk_count)
certs_dir = None
if is_https_endpoint(endpoint):
host, _ = parse_endpoint(endpoint)
certs_dir = case_workdir / "certs"
ensure_local_tls_certificates(certs_dir, host)
server_command = build_server_command(launcher, endpoint, disk_paths, certs_dir)
server_log_path = case_workdir / "server.log"
with server_log_path.open("w", encoding="utf-8") as server_log:
process = subprocess.Popen(
server_command,
cwd=str(launcher.workdir),
env=minio_env(kms_secret_key),
stdout=server_log,
stderr=subprocess.STDOUT,
text=True,
)
try:
client = S3Client(endpoint)
wait_for_minio(endpoint, process, timeout_seconds)
wait_for_s3_ready(client, timeout_seconds)
create_bucket(client, case)
request_payload = upload_case(client, case, payload_file)
head_payload = head_case(client, case)
finally:
stop_process(process)
case_dir = store_case_artifacts(
paths=paths,
case_id=case.case_id,
bucket=case.bucket,
object_name=case.object_name,
source_tree=case_workdir / "backend",
version_id=head_payload.get("VersionId") if "head_payload" in locals() else None,
request_payload=request_payload if "request_payload" in locals() else None,
head_payload=head_payload if "head_payload" in locals() else None,
plaintext_sha256=plaintext_sha256,
notes="Captured automatically from a local disposable MinIO run.",
capture_payload={
"endpoint": endpoint,
"launcher_kind": launcher.kind,
"disk_count": disk_count,
"kms_key_id": kms_secret_key.key_id,
},
)
if not preserve_workdir:
shutil.rmtree(case_workdir)
return case_dir
def select_cases(case_ids: list[str] | None) -> list[FixtureCase]:
cases = build_default_cases()
if not case_ids:
return cases
selected = [case for case in cases if case.case_id in set(case_ids)]
if len(selected) != len(set(case_ids)):
known = ", ".join(case.case_id for case in cases)
missing = sorted(set(case_ids) - {case.case_id for case in selected})
raise ValueError(f"unknown case id(s): {', '.join(missing)}; known: {known}")
return selected
def cmd_capture_matrix(args: argparse.Namespace) -> int:
paths = resolve_lab_paths(args.root)
ensure_layout(paths)
ensure_dir(args.work_root)
launcher = discover_minio_launcher(args.minio_binary, args.minio_root)
kms_secret_key = resolve_kms_secret_key(args.kms_secret_key)
cases = build_default_cases(kms_key_id=kms_secret_key.key_id)
if args.case_id:
selected_ids = set(args.case_id)
cases = [case for case in cases if case.case_id in selected_ids]
if len(cases) != len(selected_ids):
known = ", ".join(case.case_id for case in build_default_cases())
missing = sorted(selected_ids - {case.case_id for case in cases})
raise ValueError(f"unknown case id(s): {', '.join(missing)}; known: {known}")
for case in cases:
case_dir = capture_case(
paths=paths,
launcher=launcher,
endpoint=args.endpoint,
case=case,
work_root=args.work_root,
disk_count=args.disk_count,
timeout_seconds=args.timeout_seconds,
preserve_workdir=args.preserve_workdir,
kms_secret_key=kms_secret_key,
)
print(f"{case.case_id}: {case_dir}")
return 0
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Capture real MinIO backend fixtures into a stable local layout."
)
parser.set_defaults(func=None)
subparsers = parser.add_subparsers(dest="command")
init_parser = subparsers.add_parser("init", help="initialize the fixture lab root")
init_parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="fixture lab root directory")
init_parser.set_defaults(func=cmd_init)
add_case_parser = subparsers.add_parser("add-case", help="capture one fixture case")
add_case_parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="fixture lab root directory")
add_case_parser.add_argument("--case-id", required=True, help="stable fixture case identifier")
add_case_parser.add_argument("--bucket", required=True, help="bucket name")
add_case_parser.add_argument("--object", required=True, help="object key")
add_case_parser.add_argument("--version-id", help="object version id")
add_case_parser.add_argument("--source-tree", type=Path, required=True, help="backend object tree to capture")
add_case_parser.add_argument("--request-json", type=Path, help="request metadata JSON")
add_case_parser.add_argument("--head-json", type=Path, help="HEAD object metadata JSON")
add_case_parser.add_argument("--plaintext-sha256", type=Path, help="plaintext sha256 file")
add_case_parser.add_argument("--notes", help="free-form notes")
add_case_parser.set_defaults(func=cmd_add_case)
capture_parser = subparsers.add_parser(
"capture-matrix",
help="run a disposable local MinIO instance and capture the default fixture matrix",
)
capture_parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="fixture lab root directory")
capture_parser.add_argument(
"--work-root",
type=Path,
default=DEFAULT_WORK_ROOT,
help="temporary runner workspace",
)
capture_parser.add_argument(
"--minio-binary",
type=Path,
help="explicit path to a minio binary",
)
capture_parser.add_argument(
"--minio-root",
type=Path,
help="optional directory containing minio.exe",
)
capture_parser.add_argument(
"--endpoint",
default=DEFAULT_ENDPOINT,
help="local MinIO endpoint used for the disposable run",
)
capture_parser.add_argument(
"--kms-secret-key",
help=(
"static MinIO KMS secret key in the form "
"'<key-id>:<base64-32byte-key>'; overrides "
f"{LAB_KMS_SECRET_KEY_ENV}, MINIO_KMS_SECRET_KEY, and the built-in default"
),
)
capture_parser.add_argument(
"--disk-count",
type=int,
default=DEFAULT_DISK_COUNT,
help="number of backend disks to provision per case",
)
capture_parser.add_argument(
"--timeout-seconds",
type=int,
default=60,
help="startup timeout for MinIO health checks",
)
capture_parser.add_argument(
"--preserve-workdir",
action="store_true",
help="keep per-case runner work directories after capture",
)
capture_parser.add_argument(
"--case-id",
action="append",
help="capture only specific default case ids",
)
capture_parser.set_defaults(func=cmd_capture_matrix)
return parser
def main() -> int:
parser = build_parser()
args = parser.parse_args()
if args.func is None:
parser.print_help()
return 2
return args.func(args)
if __name__ == "__main__":
raise SystemExit(main())
crates/rio-v2/tests/minio_fixture_lab/test_lab.py
+207
View File
@@ -0,0 +1,207 @@
from __future__ import annotations
import importlib.util
import base64
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock
def load_lab_module():
module_path = Path(__file__).with_name("lab.py")
spec = importlib.util.spec_from_file_location("minio_fixture_lab_lab", module_path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
lab = load_lab_module()
class DiscoverMinioLauncherTests(unittest.TestCase):
def test_prefers_explicit_binary(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
binary = Path(temp_dir) / "minio.exe"
binary.write_text("", encoding="utf-8")
launcher = lab.discover_minio_launcher(explicit_binary=binary, minio_root=None)
self.assertEqual(launcher.command, [str(binary.resolve())])
self.assertEqual(launcher.workdir, binary.resolve().parent)
self.assertEqual(launcher.kind, "binary")
def test_defaults_to_bundled_binary_when_available(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
binary = Path(temp_dir) / "minio.windows-amd64.RELEASE.2025-09-07T16-13-09Z.exe"
binary.write_text("", encoding="utf-8")
original_default_binary = lab.DEFAULT_MINIO_BINARY
try:
lab.DEFAULT_MINIO_BINARY = binary
launcher = lab.discover_minio_launcher(explicit_binary=None, minio_root=None)
finally:
lab.DEFAULT_MINIO_BINARY = original_default_binary
self.assertEqual(launcher.command, [str(binary.resolve())])
self.assertEqual(launcher.workdir, binary.resolve().parent)
self.assertEqual(launcher.kind, "binary")
def test_rejects_source_checkout_without_binary(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
source_root = Path(temp_dir) / "minio-master"
source_root.mkdir()
(source_root / "main.go").write_text("package main\n", encoding="utf-8")
original_default_binary = lab.DEFAULT_MINIO_BINARY
try:
lab.DEFAULT_MINIO_BINARY = Path(temp_dir) / "missing-bundled-binary.exe"
with mock.patch.object(lab.shutil, "which", return_value=None):
with self.assertRaises(FileNotFoundError):
lab.discover_minio_launcher(explicit_binary=None, minio_root=source_root)
finally:
lab.DEFAULT_MINIO_BINARY = original_default_binary
def test_uses_binary_from_minio_root_directory(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
binary_root = Path(temp_dir) / "minio-dist"
binary_root.mkdir()
binary = binary_root / "minio.exe"
binary.write_text("", encoding="utf-8")
launcher = lab.discover_minio_launcher(explicit_binary=None, minio_root=binary_root)
self.assertEqual(launcher.command, [str(binary.resolve())])
self.assertEqual(launcher.workdir, binary.resolve().parent)
self.assertEqual(launcher.kind, "binary")
class FixtureMatrixTests(unittest.TestCase):
def test_default_matrix_covers_sse_modes_and_shapes(self) -> None:
cases = lab.build_default_cases()
case_ids = {case.case_id for case in cases}
self.assertEqual(len(cases), 6)
self.assertIn("sse-s3-singlepart-64k", case_ids)
self.assertIn("sse-kms-singlepart-64k", case_ids)
self.assertIn("sse-c-singlepart-64k", case_ids)
self.assertIn("sse-s3-multipart-8m", case_ids)
self.assertIn("sse-kms-multipart-8m", case_ids)
self.assertIn("sse-c-multipart-8m", case_ids)
def test_default_matrix_uses_configured_kms_key_id(self) -> None:
cases = lab.build_default_cases(kms_key_id="local-fixture-key")
kms_cases = [case for case in cases if case.encryption == "SSE-KMS"]
self.assertEqual(len(kms_cases), 2)
self.assertTrue(all(case.kms_key_id == "local-fixture-key" for case in kms_cases))
class RequestRecordTests(unittest.TestCase):
def test_sse_kms_request_record_includes_context_header(self) -> None:
case = lab.FixtureCase(
case_id="sse-kms-singlepart-64k",
bucket="demo",
object_name="dir/object.bin",
encryption="SSE-KMS",
size_bytes=64 * 1024,
multipart=False,
kms_key_id="minio-default-key",
kms_context={"project": "rio-v2", "stage": "fixture"},
)
record = lab.build_request_record(case)
self.assertEqual(record["headers"]["x-amz-server-side-encryption"], "aws:kms")
self.assertEqual(
record["headers"]["x-amz-server-side-encryption-aws-kms-key-id"],
"minio-default-key",
)
expected_context = base64.b64encode(
b'{"project":"rio-v2","stage":"fixture"}'
).decode("ascii")
self.assertEqual(
record["headers"]["x-amz-server-side-encryption-context"],
expected_context,
)
def test_sse_s3_request_record_uses_aes256(self) -> None:
case = lab.FixtureCase(
case_id="sse-s3-singlepart-64k",
bucket="demo",
object_name="dir/object.bin",
encryption="SSE-S3",
size_bytes=64 * 1024,
multipart=False,
)
record = lab.build_request_record(case)
self.assertEqual(record["headers"], {"x-amz-server-side-encryption": "AES256"})
def test_sse_c_request_record_includes_customer_key_headers(self) -> None:
case = lab.FixtureCase(
case_id="sse-c-singlepart-64k",
bucket="demo",
object_name="dir/object.bin",
encryption="SSE-C",
size_bytes=64 * 1024,
multipart=False,
)
record = lab.build_request_record(case)
self.assertEqual(
record["headers"]["x-amz-server-side-encryption-customer-algorithm"],
"AES256",
)
self.assertIn("x-amz-server-side-encryption-customer-key", record["headers"])
self.assertIn("x-amz-server-side-encryption-customer-key-md5", record["headers"])
class MultipartManifestTests(unittest.TestCase):
def test_complete_multipart_payload_is_xml(self) -> None:
parts = [
{"ETag": '"etag-1"', "PartNumber": 1},
{"ETag": '"etag-2"', "PartNumber": 2},
]
payload = lab.build_complete_multipart_xml(parts).decode("utf-8")
self.assertIn("<CompleteMultipartUpload>", payload)
self.assertIn("<PartNumber>1</PartNumber>", payload)
self.assertIn("<ETag>\"etag-1\"</ETag>", payload)
self.assertIn("<PartNumber>2</PartNumber>", payload)
self.assertIn("<ETag>\"etag-2\"</ETag>", payload)
class KmsSecretKeyTests(unittest.TestCase):
def test_parse_kms_secret_key_extracts_key_id(self) -> None:
setting = lab.parse_kms_secret_key("local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")
self.assertEqual(setting.secret_key, "local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")
self.assertEqual(setting.key_id, "local-fixture-key")
def test_parse_kms_secret_key_rejects_missing_separator(self) -> None:
with self.assertRaises(ValueError):
lab.parse_kms_secret_key("invalid-secret-key")
def test_minio_env_prefers_explicit_secret_key(self) -> None:
env = lab.minio_env(
kms_secret_key=lab.parse_kms_secret_key(
"local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
)
)
self.assertEqual(
env["MINIO_KMS_SECRET_KEY"],
"local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
)
if __name__ == "__main__":
unittest.main()
crates/rio-v2/tests/minio_generated_fixtures.rs
+288
View File
@@ -0,0 +1,288 @@
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
#[derive(Debug, Deserialize)]
struct RequestRecord {
encryption: String,
headers: HashMap<String, String>,
multipart: bool,
size_bytes: u64,
object: String,
}
#[derive(Debug, Deserialize)]
struct HeadRecord {
#[serde(rename = "ContentLength")]
content_length: u64,
#[serde(rename = "ServerSideEncryption")]
server_side_encryption: Option<String>,
#[serde(rename = "SSECustomerAlgorithm")]
sse_customer_algorithm: Option<String>,
#[serde(rename = "SSECustomerKeyMD5")]
sse_customer_key_md5: Option<String>,
#[serde(rename = "SSEKMSKeyId")]
sse_kms_key_id: Option<String>,
}
#[derive(Debug, Deserialize)]
struct ManifestRecord {
capture: Option<CaptureRecord>,
}
#[derive(Debug, Deserialize)]
struct CaptureRecord {
kms_key_id: Option<String>,
}
fn fixture_root() -> PathBuf {
std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/minio-generated"))
}
fn case_dir(case_id: &str) -> PathBuf {
fixture_root().join("cases").join(case_id)
}
fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> T {
let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display()))
}
fn find_object_xl_meta(case_dir: &Path) -> PathBuf {
let backend = case_dir.join("backend");
for entry in walkdir::WalkDir::new(&backend) {
let entry = entry.unwrap_or_else(|err| panic!("walk {}: {err}", backend.display()));
if !entry.file_type().is_file() {
continue;
}
if entry.file_name() != "xl.meta" {
continue;
}
let path = entry.path();
let path_text = path.to_string_lossy();
if path_text.contains(".minio.sys") {
continue;
}
return path.to_path_buf();
}
panic!("object xl.meta not found under {}", backend.display());
}
fn load_file_info(case_id: &str) -> FileInfo {
let case = case_dir(case_id);
let xl_meta = find_object_xl_meta(&case);
let data = fs::read(&xl_meta).unwrap_or_else(|err| panic!("read {}: {err}", xl_meta.display()));
get_file_info(
&data,
"fixture-bucket",
"fixture-object",
"",
FileInfoOpts {
data: false,
include_free_versions: true,
},
)
.unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta.display()))
}
fn read_manifest(case_id: &str) -> ManifestRecord {
read_json(&case_dir(case_id).join("manifest.json"))
}
fn require_generated_fixture_root() {
let root = fixture_root();
assert!(
root.join("cases").is_dir(),
"generated fixture root missing: {}. Run test/minio_fixture_lab/lab.py capture-matrix first.",
root.display()
);
}
fn metadata_keys(fi: &FileInfo) -> HashMap<&str, &str> {
fi.metadata.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect()
}
fn expected_fixture_kms_key_id(case_id: &str, request: &RequestRecord) -> Option<String> {
request
.headers
.get("x-amz-server-side-encryption-aws-kms-key-id")
.cloned()
.or_else(|| read_manifest(case_id).capture.and_then(|capture| capture.kms_key_id))
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_singlepart_sse_s3_fixture() {
require_generated_fixture_root();
let request: RequestRecord = read_json(&case_dir("sse-s3-singlepart-64k").join("request.json"));
let head: HeadRecord = read_json(&case_dir("sse-s3-singlepart-64k").join("head.json"));
let fi = load_file_info("sse-s3-singlepart-64k");
let metadata = metadata_keys(&fi);
assert_eq!(request.encryption, "SSE-S3");
assert!(!request.multipart);
assert_eq!(request.object, "dir/object.bin");
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.server_side_encryption.as_deref(), Some("AES256"));
assert_eq!(fi.parts.len(), 1);
assert_eq!(fi.parts[0].actual_size as u64, request.size_bytes);
assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"));
assert_eq!(metadata.get("content-type"), Some(&"binary/octet-stream"));
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_singlepart_sse_kms_fixture() {
require_generated_fixture_root();
let case_id = "sse-kms-singlepart-64k";
let request: RequestRecord = read_json(&case_dir(case_id).join("request.json"));
let head: HeadRecord = read_json(&case_dir(case_id).join("head.json"));
let fi = load_file_info(case_id);
let metadata = metadata_keys(&fi);
let expected_key_id = expected_fixture_kms_key_id(case_id, &request).expect("fixture kms key id");
assert_eq!(request.encryption, "SSE-KMS");
assert!(!request.multipart);
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.server_side_encryption.as_deref(), Some("aws:kms"));
assert_eq!(request.headers.get("x-amz-server-side-encryption"), Some(&"aws:kms".to_string()));
assert_eq!(request.headers.get("x-amz-server-side-encryption-aws-kms-key-id"), Some(&expected_key_id));
assert_eq!(head.sse_kms_key_id.as_deref(), Some(format!("arn:aws:kms:{expected_key_id}").as_str()));
assert_eq!(fi.parts.len(), 1);
assert_eq!(fi.parts[0].actual_size as u64, request.size_bytes);
assert_eq!(
metadata.get("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"),
Some(&expected_key_id.as_str())
);
assert_eq!(
metadata.get("X-Minio-Internal-Server-Side-Encryption-Context").copied(),
request
.headers
.get("x-amz-server-side-encryption-context")
.map(String::as_str)
);
assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key"));
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_multipart_sse_s3_fixture() {
require_generated_fixture_root();
let case_id = "sse-s3-multipart-8m";
let request: RequestRecord = read_json(&case_dir(case_id).join("request.json"));
let head: HeadRecord = read_json(&case_dir(case_id).join("head.json"));
let fi = load_file_info(case_id);
let metadata = metadata_keys(&fi);
assert_eq!(request.encryption, "SSE-S3");
assert!(request.multipart);
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.server_side_encryption.as_deref(), Some("AES256"));
assert_eq!(fi.parts.len(), 2);
assert_eq!(fi.parts.iter().map(|part| part.actual_size as u64).sum::<u64>(), request.size_bytes);
assert!(
metadata
.get("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id")
.is_some_and(|value| !value.is_empty())
);
assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"));
assert!(metadata.contains_key("X-Minio-Internal-Encrypted-Multipart"));
assert_eq!(metadata.get("X-Minio-Internal-actual-size"), Some(&"8388608"));
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_multipart_sse_kms_fixture() {
require_generated_fixture_root();
let case_id = "sse-kms-multipart-8m";
let request: RequestRecord = read_json(&case_dir(case_id).join("request.json"));
let head: HeadRecord = read_json(&case_dir(case_id).join("head.json"));
let fi = load_file_info(case_id);
let metadata = metadata_keys(&fi);
let expected_key_id = expected_fixture_kms_key_id(case_id, &request).expect("fixture kms key id");
assert_eq!(request.encryption, "SSE-KMS");
assert!(request.multipart);
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.server_side_encryption.as_deref(), Some("aws:kms"));
assert_eq!(request.headers.get("x-amz-server-side-encryption-aws-kms-key-id"), Some(&expected_key_id));
assert_eq!(head.sse_kms_key_id.as_deref(), Some(format!("arn:aws:kms:{expected_key_id}").as_str()));
assert_eq!(fi.parts.len(), 2);
assert_eq!(fi.parts.iter().map(|part| part.actual_size as u64).sum::<u64>(), request.size_bytes);
assert_eq!(
metadata.get("X-Minio-Internal-Server-Side-Encryption-Context").copied(),
request
.headers
.get("x-amz-server-side-encryption-context")
.map(String::as_str)
);
assert!(metadata.contains_key("X-Minio-Internal-Encrypted-Multipart"));
assert_eq!(metadata.get("X-Minio-Internal-actual-size"), Some(&"8388608"));
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_singlepart_sse_c_fixture() {
require_generated_fixture_root();
let case_id = "sse-c-singlepart-64k";
let request: RequestRecord = read_json(&case_dir(case_id).join("request.json"));
let head: HeadRecord = read_json(&case_dir(case_id).join("head.json"));
let fi = load_file_info(case_id);
let metadata = metadata_keys(&fi);
assert_eq!(request.encryption, "SSE-C");
assert!(!request.multipart);
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.sse_customer_algorithm.as_deref(), Some("AES256"));
assert_eq!(
head.sse_customer_key_md5.as_deref(),
request
.headers
.get("x-amz-server-side-encryption-customer-key-md5")
.map(String::as_str)
);
assert_eq!(fi.parts.len(), 1);
assert_eq!(fi.parts[0].actual_size as u64, request.size_bytes);
assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
assert!(!metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"));
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_multipart_sse_c_fixture() {
require_generated_fixture_root();
let case_id = "sse-c-multipart-8m";
let request: RequestRecord = read_json(&case_dir(case_id).join("request.json"));
let head: HeadRecord = read_json(&case_dir(case_id).join("head.json"));
let fi = load_file_info(case_id);
let metadata = metadata_keys(&fi);
assert_eq!(request.encryption, "SSE-C");
assert!(request.multipart);
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.sse_customer_algorithm.as_deref(), Some("AES256"));
assert_eq!(
head.sse_customer_key_md5.as_deref(),
request
.headers
.get("x-amz-server-side-encryption-customer-key-md5")
.map(String::as_str)
);
assert_eq!(fi.parts.len(), 2);
assert_eq!(fi.parts.iter().map(|part| part.actual_size as u64).sum::<u64>(), request.size_bytes);
assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
assert!(!metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"));
assert!(metadata.contains_key("X-Minio-Internal-Encrypted-Multipart"));
assert_eq!(metadata.get("X-Minio-Internal-actual-size"), Some(&"8388608"));
}