gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 13:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add compress support

Browse Source
This commit is contained in:
weisd
2025-06-11 17:42:45 +08:00
parent e254ddc947
commit c48ebd5149
47 changed files with 1700 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/fileinfo.rs
+23 -12
View File
@@ -1,5 +1,6 @@
use crate::error::{Error, Result};
use crate::headers::RESERVED_METADATA_PREFIX_LOWER;
use crate::headers::RUSTFS_HEALING;
use bytes::Bytes;
use rmp_serde::Serializer;
use rustfs_utils::HashAlgorithm;
@@ -9,9 +10,6 @@ use std::collections::HashMap;
use time::OffsetDateTime;
use uuid::Uuid;
use crate::headers::RESERVED_METADATA_PREFIX;
use crate::headers::RUSTFS_HEALING;
pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
@@ -24,10 +22,10 @@ pub struct ObjectPartInfo {
pub etag: String,
pub number: usize,
pub size: usize,
pub actual_size: usize, // Original data size
pub actual_size: i64, // Original data size
pub mod_time: Option<OffsetDateTime>,
// Index holds the index of the part in the erasure coding
pub index: Option<Vec<u8>>,
pub index: Option<Bytes>,
// Checksums holds checksums of the part
pub checksums: Option<HashMap<String, String>>,
}
@@ -118,15 +116,21 @@ impl ErasureInfo {
}
/// Calculate the total erasure file size for a given original size.
// Returns the final erasure size from the original size
pub fn shard_file_size(&self, total_length: usize) -> usize {
pub fn shard_file_size(&self, total_length: i64) -> i64 {
if total_length == 0 {
return 0;
}
if total_length < 0 {
return total_length;
}
let total_length = total_length as usize;
let num_shards = total_length / self.block_size;
let last_block_size = total_length % self.block_size;
let last_shard_size = calc_shard_size(last_block_size, self.data_blocks);
num_shards * self.shard_size() + last_shard_size
(num_shards * self.shard_size() + last_shard_size) as i64
}
/// Check if this ErasureInfo equals another ErasureInfo
@@ -156,7 +160,7 @@ pub struct FileInfo {
pub expire_restored: bool,
pub data_dir: Option<Uuid>,
pub mod_time: Option<OffsetDateTime>,
pub size: usize,
pub size: i64,
// File mode bits
pub mode: Option<u32>,
// WrittenByVersion is the unix time stamp of the version that created this version of the object
@@ -255,7 +259,8 @@ impl FileInfo {
etag: String,
part_size: usize,
mod_time: Option<OffsetDateTime>,
actual_size: usize,
actual_size: i64,
index: Option<Bytes>,
) {
let part = ObjectPartInfo {
etag,
@@ -263,7 +268,7 @@ impl FileInfo {
size: part_size,
mod_time,
actual_size,
index: None,
index,
checksums: None,
};
@@ -306,6 +311,12 @@ impl FileInfo {
self.metadata
.insert(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned());
}
pub fn set_data_moved(&mut self) {
self.metadata
.insert(format!("{}data-moved", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned());
}
pub fn inline_data(&self) -> bool {
self.metadata
.contains_key(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).as_str())
@@ -315,7 +326,7 @@ impl FileInfo {
/// Check if the object is compressed
pub fn is_compressed(&self) -> bool {
self.metadata
.contains_key(&format!("{}compression", RESERVED_METADATA_PREFIX))
.contains_key(&format!("{}compression", RESERVED_METADATA_PREFIX_LOWER))
}
/// Check if the object is remote (transitioned to another tier)
@@ -429,7 +440,7 @@ impl FileInfoVersions {
}
/// Calculate the total size of all versions for this object
pub fn size(&self) -> usize {
pub fn size(&self) -> i64 {
self.versions.iter().map(|v| v.size).sum()
}
}
crates/filemeta/src/filemeta.rs
+13 -9
View File
@@ -6,6 +6,7 @@ use crate::headers::{
RESERVED_METADATA_PREFIX_LOWER, VERSION_PURGE_STATUS_KEY,
};
use byteorder::ByteOrder;
use bytes::Bytes;
use rmp::Marker;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
@@ -1379,9 +1380,9 @@ pub struct MetaObject {
pub part_numbers: Vec<usize>, // Part Numbers
pub part_etags: Vec<String>, // Part ETags
pub part_sizes: Vec<usize>, // Part Sizes
pub part_actual_sizes: Vec<usize>, // Part ActualSizes (compression)
pub part_indices: Vec<Vec<u8>>, // Part Indexes (compression)
pub size: usize, // Object version size
pub part_actual_sizes: Vec<i64>, // Part ActualSizes (compression)
pub part_indices: Vec<Bytes>, // Part Indexes (compression)
pub size: i64, // Object version size
pub mod_time: Option<OffsetDateTime>, // Object version modified time
pub meta_sys: HashMap<String, Vec<u8>>, // Object version internal metadata
pub meta_user: HashMap<String, String>, // Object version metadata set by user
@@ -1538,7 +1539,7 @@ impl MetaObject {
let mut buf = vec![0u8; blen as usize];
cur.read_exact(&mut buf)?;
indices.push(buf);
indices.push(Bytes::from(buf));
}
self.part_indices = indices;
@@ -1810,13 +1811,16 @@ impl MetaObject {
}
for (k, v) in &self.meta_sys {
if k == AMZ_STORAGE_CLASS && v == b"STANDARD" {
continue;
}
if k.starts_with(RESERVED_METADATA_PREFIX)
|| k.starts_with(RESERVED_METADATA_PREFIX_LOWER)
|| k == VERSION_PURGE_STATUS_KEY
{
continue;
metadata.insert(k.to_owned(), String::from_utf8(v.to_owned()).unwrap_or_default());
}
metadata.insert(k.to_owned(), String::from_utf8(v.to_owned()).unwrap_or_default());
}
// todo: ReplicationState,Delete
@@ -2799,13 +2803,13 @@ mod test {
// 2. 测试极大的文件大小
let large_object = MetaObject {
size: usize::MAX,
size: i64::MAX,
part_sizes: vec![usize::MAX],
..Default::default()
};
// 应该能够处理大数值
assert_eq!(large_object.size, usize::MAX);
assert_eq!(large_object.size, i64::MAX);
}
#[tokio::test]
@@ -3367,7 +3371,7 @@ pub struct DetailedVersionStats {
pub free_versions: usize,
pub versions_with_data_dir: usize,
pub versions_with_inline_data: usize,
pub total_size: usize,
pub total_size: i64,
pub latest_mod_time: Option<OffsetDateTime>,
}
crates/filemeta/src/headers.rs
+2
View File
@@ -19,3 +19,5 @@ pub const X_RUSTFS_DATA_MOV: &str = "X-Rustfs-Internal-data-mov";
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
pub const AMZ_BUCKET_REPLICATION_STATUS: &str = "X-Amz-Replication-Status";
pub const AMZ_DECODED_CONTENT_LENGTH: &str = "X-Amz-Decoded-Content-Length";
pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
crates/filemeta/src/test_data.rs
+4 -4
View File
@@ -91,7 +91,7 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
let mut fm = FileMeta::new();
// 创建10个版本的对象
for i in 0..10 {
for i in 0i64..10i64 {
let version_id = Uuid::new_v4();
let data_dir = if i % 3 == 0 { Some(Uuid::new_v4()) } else { None };
@@ -113,9 +113,9 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
part_numbers: vec![1],
part_etags: vec![format!("etag-{:08x}", i)],
part_sizes: vec![1024 * (i + 1) as usize],
part_actual_sizes: vec![1024 * (i + 1) as usize],
part_actual_sizes: vec![1024 * (i + 1)],
part_indices: Vec::new(),
size: 1024 * (i + 1) as usize,
size: 1024 * (i + 1),
mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312200 + i * 60)?),
meta_sys: HashMap::new(),
meta_user: metadata,
@@ -221,7 +221,7 @@ pub fn create_xlmeta_with_inline_data() -> Result<Vec<u8>> {
part_sizes: vec![inline_data.len()],
part_actual_sizes: Vec::new(),
part_indices: Vec::new(),
size: inline_data.len(),
size: inline_data.len() as i64,
mod_time: Some(OffsetDateTime::now_utc()),
meta_sys: HashMap::new(),
meta_user: HashMap::new(),