mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 04:16:38 +00:00
feat(storage): extend PUT path tuning and observability (#3829)
* feat(storage): add multipart put stage metrics * feat(scripts): add multipart put focus runner * docs(operations): add multipart put server-path guides * chore(scripts): add local rustfs restart helper * docs(observability): add local metrics backend guide * docs(observability): add localized multipart guides * fix(ecstore): validate multipart batching path * feat(obs): add erasure encode overlap metrics * docs(ops): update overlap retest summary * docs(ops): add batchblocks retest matrix * docs(ops): extend overlap candidate summary * docs(ops): capture 8-run overlap summary * feat(storage): switch rename_data to msgpack map * test(storage): add rename_data payload checks * feat(object): add zero_copy_eager put path * docs(ops): add zero_copy_eager put guide * docs(ops): add deeper zero-copy next steps
This commit is contained in:
@@ -751,6 +751,12 @@ fn encode_msgpack<T: Serialize>(value: &T) -> Result<Vec<u8>> {
|
||||
Ok(serializer.into_inner())
|
||||
}
|
||||
|
||||
fn encode_msgpack_named<T: Serialize>(value: &T) -> Result<Vec<u8>> {
|
||||
let mut serializer = rmp_serde::Serializer::new(Vec::new()).with_struct_map();
|
||||
value.serialize(&mut serializer)?;
|
||||
Ok(serializer.into_inner())
|
||||
}
|
||||
|
||||
fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str) -> Result<T> {
|
||||
if !binary.is_empty() {
|
||||
let mut deserializer = rmp_serde::Deserializer::new(Cursor::new(binary));
|
||||
@@ -1524,6 +1530,7 @@ impl DiskAPI for RemoteDisk {
|
||||
"rename_data",
|
||||
|| async {
|
||||
let file_info = serde_json::to_string(&fi)?;
|
||||
let file_info_bin = encode_msgpack_named(&fi)?;
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
@@ -1535,6 +1542,7 @@ impl DiskAPI for RemoteDisk {
|
||||
file_info,
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
file_info_bin: file_info_bin.into(),
|
||||
});
|
||||
|
||||
let response = client.rename_data(request).await?.into_inner();
|
||||
@@ -1543,7 +1551,8 @@ impl DiskAPI for RemoteDisk {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
let rename_data_resp = serde_json::from_str::<RenameDataResp>(&response.rename_data_resp)?;
|
||||
let rename_data_resp =
|
||||
decode_msgpack_or_json::<RenameDataResp>(&response.rename_data_resp_bin, &response.rename_data_resp)?;
|
||||
|
||||
Ok(rename_data_resp)
|
||||
},
|
||||
@@ -2373,6 +2382,64 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_rename_data_file_info() -> FileInfo {
|
||||
FileInfo {
|
||||
volume: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
data_dir: Some(Uuid::new_v4()),
|
||||
size: 64 * 1024,
|
||||
mod_time: Some(::time::OffsetDateTime::UNIX_EPOCH + ::time::Duration::seconds(1)),
|
||||
metadata: [
|
||||
("etag".to_string(), "etag-value".to_string()),
|
||||
("content-type".to_string(), "application/octet-stream".to_string()),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
erasure: rustfs_filemeta::ErasureInfo {
|
||||
algorithm: rustfs_filemeta::ERASURE_ALGORITHM.to_string(),
|
||||
data_blocks: 4,
|
||||
parity_blocks: 2,
|
||||
block_size: 1024 * 1024,
|
||||
index: 1,
|
||||
distribution: vec![1, 2, 3, 4, 5, 6],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_file_info_named_msgpack_is_smaller_than_json() {
|
||||
let file_info = sample_rename_data_file_info();
|
||||
let json = serde_json::to_vec(&file_info).expect("file info json should encode");
|
||||
let named_msgpack = encode_msgpack_named(&file_info).expect("file info named msgpack should encode");
|
||||
|
||||
assert!(
|
||||
named_msgpack.len() < json.len(),
|
||||
"expected named msgpack payload to be smaller than json (msgpack={}, json={})",
|
||||
named_msgpack.len(),
|
||||
json.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_resp_named_msgpack_is_smaller_than_json() {
|
||||
let response = RenameDataResp {
|
||||
old_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![1_u8; 32]),
|
||||
};
|
||||
let json = serde_json::to_vec(&response).expect("rename data response json should encode");
|
||||
let named_msgpack = encode_msgpack_named(&response).expect("rename data response named msgpack should encode");
|
||||
|
||||
assert!(
|
||||
named_msgpack.len() < json.len(),
|
||||
"expected named msgpack payload to be smaller than json (msgpack={}, json={})",
|
||||
named_msgpack.len(),
|
||||
json.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct SinkTestWriter;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user