gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 06:13:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

change objectinfo metadata use HashMap

Browse Source 
add copyobject compresss support

fix listobject size use actual_size
This commit is contained in:
weisd
2025-06-23 11:37:42 +08:00
parent dc93398768
commit 34ec13e532
9 changed files with 143 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/storage/ecfs.rs
+66 -35
View File
@@ -67,6 +67,7 @@ use query::instance::make_rustfsms;
use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_filemeta::headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING};
use rustfs_rio::CompressReader;
use rustfs_rio::EtagReader;
use rustfs_rio::HashReader;
use rustfs_rio::Reader;
use rustfs_rio::WarpReader;
@@ -329,7 +330,7 @@ impl S3 for FS {
..Default::default()
};
let dst_opts = copy_dst_opts(&bucket, &key, version_id, &req.headers, None)
let dst_opts = copy_dst_opts(&bucket, &key, version_id, &req.headers, HashMap::new())
.await
.map_err(ApiError::from)?;
@@ -356,13 +357,50 @@ impl S3 for FS {
src_info.metadata_only = true;
}
let reader = Box::new(WarpReader::new(gr.stream));
let hrd = HashReader::new(reader, gr.object_info.size, gr.object_info.size, None, false).map_err(ApiError::from)?;
let mut reader: Box<dyn Reader> = Box::new(WarpReader::new(gr.stream));
let actual_size = src_info.get_actual_size().map_err(ApiError::from)?;
let mut length = actual_size;
let mut compress_metadata = HashMap::new();
if is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64 {
compress_metadata.insert(
format!("{}compression", RESERVED_METADATA_PREFIX_LOWER),
CompressionAlgorithm::default().to_string(),
);
compress_metadata.insert(format!("{}actual-size", RESERVED_METADATA_PREFIX_LOWER,), actual_size.to_string());
let hrd = EtagReader::new(reader, None);
// let hrd = HashReader::new(reader, length, actual_size, None, false).map_err(ApiError::from)?;
reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default()));
length = -1;
} else {
src_info
.user_defined
.remove(&format!("{}compression", RESERVED_METADATA_PREFIX_LOWER));
src_info
.user_defined
.remove(&format!("{}actual-size", RESERVED_METADATA_PREFIX_LOWER));
src_info
.user_defined
.remove(&format!("{}compression-size", RESERVED_METADATA_PREFIX_LOWER));
}
let hrd = HashReader::new(reader, length, actual_size, None, false).map_err(ApiError::from)?;
src_info.put_object_reader = Some(PutObjReader::new(hrd));
// check quota
// TODO: src metadada
for (k, v) in compress_metadata {
src_info.user_defined.insert(k, v);
}
// TODO: src tags
let oi = store
@@ -417,7 +455,7 @@ impl S3 for FS {
let metadata = extract_metadata(&req.headers);
let opts: ObjectOptions = del_opts(&bucket, &key, version_id, &req.headers, Some(metadata))
let opts: ObjectOptions = del_opts(&bucket, &key, version_id, &req.headers, metadata)
.await
.map_err(ApiError::from)?;
@@ -487,7 +525,7 @@ impl S3 for FS {
let metadata = extract_metadata(&req.headers);
let opts: ObjectOptions = del_opts(&bucket, "", None, &req.headers, Some(metadata))
let opts: ObjectOptions = del_opts(&bucket, "", None, &req.headers, metadata)
.await
.map_err(ApiError::from)?;
@@ -724,7 +762,7 @@ impl S3 for FS {
content_type,
last_modified,
e_tag: info.etag,
metadata,
metadata: Some(metadata),
version_id: info.version_id.map(|v| v.to_string()),
// metadata: object_metadata,
..Default::default()
@@ -844,7 +882,7 @@ impl S3 for FS {
let mut obj = Object {
key: Some(v.name.to_owned()),
last_modified: v.mod_time.map(Timestamp::from),
size: Some(v.size),
size: Some(v.get_actual_size().unwrap_or_default()),
e_tag: v.etag.clone(),
..Default::default()
};
@@ -1043,7 +1081,7 @@ impl S3 for FS {
let mt = metadata.clone();
let mt2 = metadata.clone();
let mut opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(mt))
let mut opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, mt)
.await
.map_err(ApiError::from)?;
@@ -1053,14 +1091,12 @@ impl S3 for FS {
let dsc = must_replicate(&bucket, &key, &repoptions).await;
// warn!("dsc {}", &dsc.replicate_any().clone());
if dsc.replicate_any() {
if let Some(metadata) = opts.user_defined.as_mut() {
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp");
let now: DateTime<Utc> = Utc::now();
let formatted_time = now.to_rfc3339();
metadata.insert(k, formatted_time);
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status");
metadata.insert(k, dsc.pending_status());
}
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp");
let now: DateTime<Utc> = Utc::now();
let formatted_time = now.to_rfc3339();
opts.user_defined.insert(k, formatted_time);
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status");
opts.user_defined.insert(k, dsc.pending_status());
}
let obj_info = store
@@ -1121,7 +1157,7 @@ impl S3 for FS {
);
}
let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata))
let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, metadata)
.await
.map_err(ApiError::from)?;
@@ -2255,11 +2291,10 @@ impl S3 for FS {
s3_error!(InternalError, "{}", e.to_string())
})?;
let legal_hold = if let Some(ud) = object_info.user_defined {
ud.get("x-amz-object-lock-legal-hold").map(|v| v.as_str().to_string())
} else {
None
};
let legal_hold = object_info
.user_defined
.get("x-amz-object-lock-legal-hold")
.map(|v| v.as_str().to_string());
let status = if let Some(v) = legal_hold {
v
@@ -2356,20 +2391,16 @@ impl S3 for FS {
s3_error!(InternalError, "{}", e.to_string())
})?;
let mode = if let Some(ref ud) = object_info.user_defined {
ud.get("x-amz-object-lock-mode")
.map(|v| ObjectLockRetentionMode::from(v.as_str().to_string()))
} else {
None
};
let mode = object_info
.user_defined
.get("x-amz-object-lock-mode")
.map(|v| ObjectLockRetentionMode::from(v.as_str().to_string()));
let retain_until_date = if let Some(ref ud) = object_info.user_defined {
ud.get("x-amz-object-lock-retain-until-date")
.and_then(|v| OffsetDateTime::parse(v.as_str(), &Rfc3339).ok())
.map(Timestamp::from)
} else {
None
};
let retain_until_date = object_info
.user_defined
.get("x-amz-object-lock-retain-until-date")
.and_then(|v| OffsetDateTime::parse(v.as_str(), &Rfc3339).ok())
.map(Timestamp::from);
Ok(S3Response::new(GetObjectRetentionOutput {
retention: Some(ObjectLockRetention { mode, retain_until_date }),
rustfs/src/storage/options.rs
+29 -32
View File
@@ -14,7 +14,7 @@ pub async fn del_opts(
object: &str,
vid: Option<String>,
headers: &HeaderMap<HeaderValue>,
metadata: Option<HashMap<String, String>>,
metadata: HashMap<String, String>,
) -> Result<ObjectOptions> {
let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
let version_suspended = BucketVersioningSys::suspended(bucket).await;
@@ -33,7 +33,7 @@ pub async fn del_opts(
}
}
let mut opts = put_opts_from_headers(headers, metadata)
let mut opts = put_opts_from_headers(headers, metadata.clone())
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
opts.version_id = {
@@ -72,7 +72,7 @@ pub async fn get_opts(
}
}
let mut opts = get_default_opts(headers, None, false)
let mut opts = get_default_opts(headers, HashMap::new(), false)
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
opts.version_id = {
@@ -97,7 +97,7 @@ pub async fn put_opts(
object: &str,
vid: Option<String>,
headers: &HeaderMap<HeaderValue>,
metadata: Option<HashMap<String, String>>,
metadata: HashMap<String, String>,
) -> Result<ObjectOptions> {
let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
@@ -136,30 +136,27 @@ pub async fn copy_dst_opts(
object: &str,
vid: Option<String>,
headers: &HeaderMap<HeaderValue>,
metadata: Option<HashMap<String, String>>,
metadata: HashMap<String, String>,
) -> Result<ObjectOptions> {
put_opts(bucket, object, vid, headers, metadata).await
}
pub fn copy_src_opts(_bucket: &str, _object: &str, headers: &HeaderMap<HeaderValue>) -> Result<ObjectOptions> {
get_default_opts(headers, None, false)
get_default_opts(headers, HashMap::new(), false)
}
pub fn put_opts_from_headers(
headers: &HeaderMap<HeaderValue>,
metadata: Option<HashMap<String, String>>,
) -> Result<ObjectOptions> {
pub fn put_opts_from_headers(headers: &HeaderMap<HeaderValue>, metadata: HashMap<String, String>) -> Result<ObjectOptions> {
get_default_opts(headers, metadata, false)
}
/// Creates default options for getting an object from a bucket.
pub fn get_default_opts(
_headers: &HeaderMap<HeaderValue>,
metadata: Option<HashMap<String, String>>,
metadata: HashMap<String, String>,
_copy_source: bool,
) -> Result<ObjectOptions> {
Ok(ObjectOptions {
user_defined: metadata.clone(),
user_defined: metadata,
..Default::default()
})
}
@@ -244,13 +241,13 @@ mod tests {
#[tokio::test]
async fn test_del_opts_basic() {
let headers = create_test_headers();
let metadata = Some(create_test_metadata());
let metadata = create_test_metadata();
let result = del_opts("test-bucket", "test-object", None, &headers, metadata).await;
assert!(result.is_ok());
let opts = result.unwrap();
assert!(opts.user_defined.is_some());
assert!(!opts.user_defined.is_empty());
assert_eq!(opts.version_id, None);
}
@@ -258,7 +255,7 @@ mod tests {
async fn test_del_opts_with_directory_object() {
let headers = create_test_headers();
let result = del_opts("test-bucket", "test-dir/", None, &headers, None).await;
let result = del_opts("test-bucket", "test-dir/", None, &headers, HashMap::new()).await;
assert!(result.is_ok());
let opts = result.unwrap();
@@ -270,7 +267,7 @@ mod tests {
let headers = create_test_headers();
let valid_uuid = Uuid::new_v4().to_string();
let result = del_opts("test-bucket", "test-object", Some(valid_uuid.clone()), &headers, None).await;
let result = del_opts("test-bucket", "test-object", Some(valid_uuid.clone()), &headers, HashMap::new()).await;
// This test may fail if versioning is not enabled for the bucket
// In a real test environment, you would mock BucketVersioningSys
@@ -289,7 +286,7 @@ mod tests {
let headers = create_test_headers();
let invalid_uuid = "invalid-uuid".to_string();
let result = del_opts("test-bucket", "test-object", Some(invalid_uuid), &headers, None).await;
let result = del_opts("test-bucket", "test-object", Some(invalid_uuid), &headers, HashMap::new()).await;
assert!(result.is_err());
if let Err(err) = result {
@@ -361,13 +358,13 @@ mod tests {
#[tokio::test]
async fn test_put_opts_basic() {
let headers = create_test_headers();
let metadata = Some(create_test_metadata());
let metadata = create_test_metadata();
let result = put_opts("test-bucket", "test-object", None, &headers, metadata).await;
assert!(result.is_ok());
let opts = result.unwrap();
assert!(opts.user_defined.is_some());
assert!(!opts.user_defined.is_empty());
assert_eq!(opts.version_id, None);
}
@@ -375,7 +372,7 @@ mod tests {
async fn test_put_opts_with_directory_object() {
let headers = create_test_headers();
let result = put_opts("test-bucket", "test-dir/", None, &headers, None).await;
let result = put_opts("test-bucket", "test-dir/", None, &headers, HashMap::new()).await;
assert!(result.is_ok());
let opts = result.unwrap();
@@ -387,7 +384,7 @@ mod tests {
let headers = create_test_headers();
let invalid_uuid = "invalid-uuid".to_string();
let result = put_opts("test-bucket", "test-object", Some(invalid_uuid), &headers, None).await;
let result = put_opts("test-bucket", "test-object", Some(invalid_uuid), &headers, HashMap::new()).await;
assert!(result.is_err());
if let Err(err) = result {
@@ -405,13 +402,13 @@ mod tests {
#[tokio::test]
async fn test_copy_dst_opts() {
let headers = create_test_headers();
let metadata = Some(create_test_metadata());
let metadata = create_test_metadata();
let result = copy_dst_opts("test-bucket", "test-object", None, &headers, metadata).await;
assert!(result.is_ok());
let opts = result.unwrap();
assert!(opts.user_defined.is_some());
assert!(!opts.user_defined.is_empty());
}
#[test]
@@ -422,20 +419,20 @@ mod tests {
assert!(result.is_ok());
let opts = result.unwrap();
assert!(opts.user_defined.is_none());
assert!(opts.user_defined.is_empty());
}
#[test]
fn test_put_opts_from_headers() {
let headers = create_test_headers();
let metadata = Some(create_test_metadata());
let metadata = create_test_metadata();
let result = put_opts_from_headers(&headers, metadata);
assert!(result.is_ok());
let opts = result.unwrap();
assert!(opts.user_defined.is_some());
let user_defined = opts.user_defined.unwrap();
assert!(!opts.user_defined.is_empty());
let user_defined = opts.user_defined;
assert_eq!(user_defined.get("key1"), Some(&"value1".to_string()));
assert_eq!(user_defined.get("key2"), Some(&"value2".to_string()));
}
@@ -443,14 +440,14 @@ mod tests {
#[test]
fn test_get_default_opts_with_metadata() {
let headers = create_test_headers();
let metadata = Some(create_test_metadata());
let metadata = create_test_metadata();
let result = get_default_opts(&headers, metadata, false);
assert!(result.is_ok());
let opts = result.unwrap();
assert!(opts.user_defined.is_some());
let user_defined = opts.user_defined.unwrap();
assert!(!opts.user_defined.is_empty());
let user_defined = opts.user_defined;
assert_eq!(user_defined.get("key1"), Some(&"value1".to_string()));
assert_eq!(user_defined.get("key2"), Some(&"value2".to_string()));
}
@@ -459,11 +456,11 @@ mod tests {
fn test_get_default_opts_without_metadata() {
let headers = create_test_headers();
let result = get_default_opts(&headers, None, false);
let result = get_default_opts(&headers, HashMap::new(), false);
assert!(result.is_ok());
let opts = result.unwrap();
assert!(opts.user_defined.is_none());
assert!(opts.user_defined.is_empty());
}
#[test]