gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 03:22:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): optimize ObjectInfo clone and fix critical TODOs (#3149)

Browse Source 
* fix(ecstore): optimize ObjectInfo clone and fix critical TODOs

## ObjectInfo hot-path clone optimization (issue #653 item 2)
- Wrap user_defined (HashMap), user_tags (String), parts (Vec) in Arc
- Clone cost reduced from ~20 heap allocations to O(1) ref count bump
- Updated 11 downstream access sites with explicit deref where needed

## Critical TODO/FIXME fixes (issue #653 item 7)
- rpc/peer_s3_client.rs: descriptive error message for empty peer response
- set_disk/write.rs: concurrent rollback deletes via tokio::spawn + join_all
- store_list_objects.rs: resolved FIXME with explanation + 4 regression tests
- store/bucket.rs: namespace write locks for make_bucket and delete_bucket

Skipped TODOs (too risky without broader context):
- multipart.rs:30 nslock — causes lock timeout in existing tests
- bucket.rs:88 cached list_bucket — needs cache invalidation strategy
- bucket.rs:149 replication delete — needs replication subsystem integration

1134 tests pass (1 flaky under parallel execution, passes individually).

* fix: address PR #3149 review comments

- Preserve NamespaceLockQuorumUnavailable variant in bucket create/delete
- Fix cancellation test to actually test cancellation (keep senders open)
- Update ObjectInfo consumers in app/ for Arc-backed fields
- Update bucket_usecase.rs user_tags to Arc<String>

* fix: update ObjectInfo Arc consumers

* fix: address ecstore merge review comments

* fix: cancel blocked merge output sends

* fix(ecstore): satisfy ObjectInfo clone clippy
This commit is contained in:
安正超
2026-06-01 08:44:58 +08:00
committed by GitHub
parent bd571a575f
commit 97dd825468
19 changed files with 479 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/app/bucket_usecase.rs
+12 -11
View File
@@ -472,7 +472,7 @@ fn build_list_object_versions_m_output(
None
};
let user_tags = if permission.tags_allowed && !object.user_tags.is_empty() {
Some(object.user_tags.clone())
Some((*object.user_tags).clone())
} else {
None
};
@@ -569,7 +569,7 @@ fn build_list_objects_v2m_output(
None
};
let user_tags = if permission.tags_allowed && !object.user_tags.is_empty() {
Some(object.user_tags.clone())
Some((*object.user_tags).clone())
} else {
None
};
@@ -2101,6 +2101,7 @@ impl DefaultBucketUsecase {
mod tests {
use super::*;
use http::{Extensions, HeaderMap, Method, Uri};
use std::sync::Arc;
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
@@ -2717,11 +2718,11 @@ mod tests {
name: "obj-a".to_string(),
mod_time: Some(datetime!(2025-01-01 00:00 UTC)),
size: 11,
user_defined: HashMap::from([("project".to_string(), "alpha".to_string())]),
user_defined: Arc::new(HashMap::from([("project".to_string(), "alpha".to_string())])),
parity_blocks: 2,
data_blocks: 4,
version_id: Some(Uuid::nil()),
user_tags: "env=prod".to_string(),
user_tags: Arc::new("env=prod".to_string()),
is_latest: true,
etag: Some("0123456789abcdef0123456789abcdef".to_string()),
..Default::default()
@@ -2731,7 +2732,7 @@ mod tests {
name: "obj-b".to_string(),
mod_time: Some(datetime!(2025-01-02 00:00 UTC)),
delete_marker: true,
user_defined: HashMap::from([("marker".to_string(), "true".to_string())]),
user_defined: Arc::new(HashMap::from([("marker".to_string(), "true".to_string())])),
version_id: None,
..Default::default()
},
@@ -2825,8 +2826,8 @@ mod tests {
name: "logs and more/object one.txt".to_string(),
mod_time: Some(datetime!(2025-01-04 00:00 UTC)),
size: 7,
user_defined: HashMap::from([("secret".to_string(), "value".to_string())]),
user_tags: "env=prod".to_string(),
user_defined: Arc::new(HashMap::from([("secret".to_string(), "value".to_string())])),
user_tags: Arc::new("env=prod".to_string()),
parity_blocks: 1,
data_blocks: 2,
..Default::default()
@@ -2905,10 +2906,10 @@ mod tests {
name: "logs/obj a.txt".to_string(),
mod_time: Some(datetime!(2025-01-03 00:00 UTC)),
size: 11,
user_defined: HashMap::from([("project".to_string(), "alpha".to_string())]),
user_defined: Arc::new(HashMap::from([("project".to_string(), "alpha".to_string())])),
parity_blocks: 2,
data_blocks: 4,
user_tags: "env=prod".to_string(),
user_tags: Arc::new("env=prod".to_string()),
etag: Some("0123456789abcdef0123456789abcdef".to_string()),
..Default::default()
}],
@@ -2981,8 +2982,8 @@ mod tests {
name: "logs and more/object one.txt".to_string(),
mod_time: Some(datetime!(2025-01-05 00:00 UTC)),
size: 13,
user_defined: HashMap::from([("secret".to_string(), "value".to_string())]),
user_tags: "env=prod".to_string(),
user_defined: Arc::new(HashMap::from([("secret".to_string(), "value".to_string())])),
user_tags: Arc::new("env=prod".to_string()),
parity_blocks: 1,
data_blocks: 2,
..Default::default()
rustfs/src/app/object_usecase.rs
+23 -18
View File
@@ -559,7 +559,7 @@ fn delete_replication_state_from_config(
) -> Option<ReplicationState> {
let opts = ReplicationObjectOpts {
name: obj_info.name.clone(),
user_tags: obj_info.user_tags.clone(),
user_tags: (*obj_info.user_tags).clone(),
version_id,
delete_marker: obj_info.delete_marker,
op_type: ReplicationType::Delete,
@@ -1347,7 +1347,7 @@ impl DefaultObjectUsecase {
check_preconditions(&req.headers, &info)?;
debug!(object_size = info.size, part_count = info.parts.len(), "GET object metadata snapshot");
for part in &info.parts {
for part in info.parts.iter() {
debug!(
part_number = part.number,
part_size = part.size,
@@ -2810,7 +2810,10 @@ impl DefaultObjectUsecase {
src_info.metadata_only = true;
}
strip_managed_encryption_metadata(&mut src_info.user_defined);
// Extract user_defined from Arc for mutation; it will be re-wrapped after all edits.
let mut user_defined = (*src_info.user_defined).clone();
strip_managed_encryption_metadata(&mut user_defined);
let actual_size = src_info.get_actual_size().map_err(ApiError::from)?;
@@ -2824,27 +2827,27 @@ impl DefaultObjectUsecase {
insert_str(&mut compress_metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string());
insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
} else {
remove_str(&mut src_info.user_defined, SUFFIX_COMPRESSION);
remove_str(&mut src_info.user_defined, SUFFIX_ACTUAL_SIZE);
remove_str(&mut src_info.user_defined, SUFFIX_COMPRESSION_SIZE);
remove_str(&mut user_defined, SUFFIX_COMPRESSION);
remove_str(&mut user_defined, SUFFIX_ACTUAL_SIZE);
remove_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE);
}
// Handle MetadataDirective REPLACE: replace user metadata while preserving system metadata.
// System metadata (compression, encryption) is added after this block to ensure
// it's not cleared by the REPLACE operation.
if metadata_directive.as_ref().map(|d| d.as_str()) == Some(MetadataDirective::REPLACE) {
src_info.user_defined.clear();
user_defined.clear();
if let Some(metadata) = metadata {
src_info.user_defined.extend(metadata);
user_defined.extend(metadata);
}
if let Some(ct) = content_type {
src_info.content_type = Some(ct.clone());
src_info.user_defined.insert("content-type".to_string(), ct);
user_defined.insert("content-type".to_string(), ct);
}
}
let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some();
remove_object_lock_metadata_for_copy(&mut src_info.user_defined);
remove_object_lock_metadata_for_copy(&mut user_defined);
if let Some(object_lock_metadata) = build_put_like_object_lock_metadata(
&bucket,
object_lock_legal_hold_status,
@@ -2853,9 +2856,9 @@ impl DefaultObjectUsecase {
)
.await?
{
src_info.user_defined.extend(object_lock_metadata);
user_defined.extend(object_lock_metadata);
}
apply_bucket_default_lock_retention(&bucket, &mut src_info.user_defined, has_explicit_object_lock_retention).await?;
apply_bucket_default_lock_retention(&bucket, &mut user_defined, has_explicit_object_lock_retention).await?;
let mut reader = if should_compress {
let hrd = HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)?;
@@ -2892,7 +2895,7 @@ impl DefaultObjectUsecase {
reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
src_info.user_defined.extend(encryption_material_to_metadata(&material));
user_defined.extend(encryption_material_to_metadata(&material));
}
src_info.put_object_reader = Some(PutObjReader::new(reader));
@@ -2900,9 +2903,11 @@ impl DefaultObjectUsecase {
// check quota
for (k, v) in compress_metadata {
src_info.user_defined.insert(k, v);
user_defined.insert(k, v);
}
src_info.user_defined = Arc::new(user_defined);
self.check_bucket_quota(&bucket, QuotaOperation::CopyObject, src_info.size as u64)
.await?;
let has_bucket_metadata = self.bucket_metadata_sys().is_some();
@@ -3899,7 +3904,7 @@ impl DefaultObjectUsecase {
}
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), *rreq.days.as_ref().unwrap_or(&1));
let mut metadata = obj_info.user_defined.clone();
let mut metadata = (*obj_info.user_defined).clone();
let mut header = HeaderMap::new();
@@ -3931,7 +3936,7 @@ impl DefaultObjectUsecase {
.to_string(),
);
}
obj_info.user_defined = metadata;
obj_info.user_defined = Arc::new(metadata);
store
.clone()
@@ -5032,7 +5037,7 @@ mod tests {
let metadata = HashMap::new();
let standard_info = ObjectInfo {
storage_class: Some(storageclass::STANDARD.to_string()),
user_defined: metadata.clone(),
user_defined: Arc::new(metadata.clone()),
..Default::default()
};
assert!(response_storage_class(&standard_info, &metadata).is_none());
@@ -5041,7 +5046,7 @@ mod tests {
metadata.insert(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string());
let infrequent_access_info = ObjectInfo {
storage_class: Some(storageclass::STANDARD_IA.to_string()),
user_defined: metadata.clone(),
user_defined: Arc::new(metadata.clone()),
..Default::default()
};
assert_eq!(