gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 21:26:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Introduce content checksums and improve multipart/object metadata handling (#671)

Browse Source 
* feat:  adapt to s3s typed etag support

* refactor: move replication struct to rustfs_filemeta, fix filemeta transition bug

* add head_object checksum, filter object metadata output

* fix multipart checksum

* fix multipart checksum

* add content md5,sha256 check

* fix test

* fix cargo

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>
This commit is contained in:
weisd
2025-10-20 23:46:13 +08:00
committed by GitHub
parent 46797dc815
commit cd1e244c68
56 changed files with 3331 additions and 810 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk.rs
+114 -83
View File
@@ -72,13 +72,13 @@ use rustfs_filemeta::{
};
use rustfs_lock::fast_lock::types::LockResult;
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_rio::{EtagResolvable, HashReader, TryGetIndex as _, WarpReader};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _, WarpReader};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use rustfs_utils::http::headers::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_utils::{
HashAlgorithm,
crypto::{base64_decode, base64_encode, hex},
crypto::hex,
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
};
use rustfs_workers::workers::Workers;
@@ -158,10 +158,7 @@ impl SetDisks {
LockResult::Conflict {
current_owner,
current_mode,
} => format!(
"{mode} lock conflicted on {bucket}/{object}: held by {current_owner} as {:?}",
current_mode
),
} => format!("{mode} lock conflicted on {bucket}/{object}: held by {current_owner} as {current_mode:?}"),
LockResult::Acquired => format!("unexpected lock state while acquiring {mode} lock on {bucket}/{object}"),
}
}
@@ -922,9 +919,8 @@ impl SetDisks {
}
fn get_upload_id_dir(bucket: &str, object: &str, upload_id: &str) -> String {
// warn!("get_upload_id_dir upload_id {:?}", upload_id);
let upload_uuid = base64_decode(upload_id.as_bytes())
let upload_uuid = base64_simd::URL_SAFE_NO_PAD
.decode_to_vec(upload_id.as_bytes())
.and_then(|v| {
String::from_utf8(v).map_or(Ok(upload_id.to_owned()), |v| {
let parts: Vec<_> = v.splitn(2, '.').collect();
@@ -2950,6 +2946,7 @@ impl SetDisks {
part.mod_time,
part.actual_size,
part.index.clone(),
part.checksums.clone(),
);
if is_inline_buffer {
if let Some(writer) = writers[index].take() {
@@ -3528,9 +3525,9 @@ impl ObjectIO for SetDisks {
// }
if object_info.size == 0 {
if let Some(rs) = range {
let _ = rs.get_offset_length(object_info.size)?;
}
// if let Some(rs) = range {
// let _ = rs.get_offset_length(object_info.size)?;
// }
let reader = GetObjectReader {
stream: Box::new(Cursor::new(Vec::new())),
@@ -3712,7 +3709,7 @@ impl ObjectIO for SetDisks {
let stream = mem::replace(
&mut data.stream,
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, false)?,
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, None, false)?,
);
let (reader, w_size) = match Arc::new(erasure).encode(stream, &mut writers, write_quorum).await {
@@ -3729,7 +3726,12 @@ impl ObjectIO for SetDisks {
// }
if (w_size as i64) < data.size() {
return Err(Error::other("put_object write size < data.size()"));
warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size());
return Err(Error::other(format!(
"put_object write size < data.size(), w_size={}, data.size={}",
w_size,
data.size()
)));
}
if user_defined.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression")) {
@@ -3756,31 +3758,42 @@ impl ObjectIO for SetDisks {
}
}
if fi.checksum.is_none() {
if let Some(content_hash) = data.as_hash_reader().content_hash() {
fi.checksum = Some(content_hash.to_bytes(&[]));
}
}
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS) {
if sc == storageclass::STANDARD {
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
}
}
let now = OffsetDateTime::now_utc();
let mod_time = if let Some(mod_time) = opts.mod_time {
Some(mod_time)
} else {
Some(OffsetDateTime::now_utc())
};
for (i, fi) in parts_metadatas.iter_mut().enumerate() {
fi.metadata = user_defined.clone();
for (i, pfi) in parts_metadatas.iter_mut().enumerate() {
pfi.metadata = user_defined.clone();
if is_inline_buffer {
if let Some(writer) = writers[i].take() {
fi.data = Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default());
pfi.data = Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default());
}
fi.set_inline_data();
pfi.set_inline_data();
}
fi.mod_time = Some(now);
fi.size = w_size as i64;
fi.versioned = opts.versioned || opts.version_suspended;
fi.add_object_part(1, etag.clone(), w_size, fi.mod_time, actual_size, index_op.clone());
pfi.mod_time = mod_time;
pfi.size = w_size as i64;
pfi.versioned = opts.versioned || opts.version_suspended;
pfi.add_object_part(1, etag.clone(), w_size, mod_time, actual_size, index_op.clone(), None);
pfi.checksum = fi.checksum.clone();
if opts.data_movement {
fi.set_data_moved();
pfi.set_data_moved();
}
}
@@ -3815,7 +3828,8 @@ impl ObjectIO for SetDisks {
fi.replication_state_internal = Some(opts.put_replication_state());
// TODO: version support
fi.is_latest = true;
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
}
}
@@ -4430,8 +4444,6 @@ impl StorageAPI for SetDisks {
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
// warn!("get object_info fi {:?}", &fi);
let oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
Ok(oi)
@@ -4759,6 +4771,11 @@ impl StorageAPI for SetDisks {
uploaded_parts.push(CompletePart {
part_num: p_info.part_num,
etag: p_info.etag,
checksum_crc32: None,
checksum_crc32c: None,
checksum_sha1: None,
checksum_sha256: None,
checksum_crc64nvme: None,
});
}
if let Err(err) = self.complete_multipart_upload(bucket, object, &res.upload_id, uploaded_parts, &ObjectOptions {
@@ -4834,64 +4851,24 @@ impl StorageAPI for SetDisks {
let write_quorum = fi.write_quorum(self.default_write_quorum());
let disks = self.disks.read().await;
if let Some(checksum) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
&& !checksum.is_empty()
&& data
.as_hash_reader()
.content_crc_type()
.is_none_or(|v| v.to_string() != *checksum)
{
return Err(Error::other(format!("checksum mismatch: {checksum}")));
}
let disks = self.disks.read().await.clone();
let disks = disks.clone();
let shuffle_disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
let part_suffix = format!("part.{part_id}");
let tmp_part = format!("{}x{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp());
let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}"));
// let mut writers = Vec::with_capacity(disks.len());
// let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
// let shared_size = erasure.shard_size(erasure.block_size);
// let futures = disks.iter().map(|disk| {
// let disk = disk.clone();
// let tmp_part_path = tmp_part_path.clone();
// tokio::spawn(async move {
// if let Some(disk) = disk {
// // let writer = disk.append_file(RUSTFS_META_TMP_BUCKET, &tmp_part_path).await?;
// // let filewriter = disk
// // .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, data.content_length)
// // .await?;
// match new_bitrot_filewriter(
// disk.clone(),
// RUSTFS_META_TMP_BUCKET,
// &tmp_part_path,
// false,
// DEFAULT_BITROT_ALGO,
// shared_size,
// )
// .await
// {
// Ok(writer) => Ok(Some(writer)),
// Err(e) => Err(e),
// }
// } else {
// Ok(None)
// }
// })
// });
// for x in join_all(futures).await {
// let x = x??;
// writers.push(x);
// }
// let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
// let stream = replace(&mut data.stream, Box::new(empty()));
// let etag_stream = EtagReader::new(stream);
// let (w_size, mut etag) = Arc::new(erasure)
// .encode(etag_stream, &mut writers, data.content_length, write_quorum)
// .await?;
// if let Err(err) = close_bitrot_writers(&mut writers).await {
// error!("close_bitrot_writers err {:?}", err);
// }
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let mut writers = Vec::with_capacity(shuffle_disks.len());
@@ -4944,7 +4921,7 @@ impl StorageAPI for SetDisks {
let stream = mem::replace(
&mut data.stream,
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, false)?,
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, None, false)?,
);
let (reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: 出错，删除临时目录
@@ -4952,7 +4929,12 @@ impl StorageAPI for SetDisks {
let _ = mem::replace(&mut data.stream, reader);
if (w_size as i64) < data.size() {
return Err(Error::other("put_object_part write size < data.size()"));
warn!("put_object_part write size < data.size(), w_size={}, data.size={}", w_size, data.size());
return Err(Error::other(format!(
"put_object_part write size < data.size(), w_size={}, data.size={}",
w_size,
data.size()
)));
}
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
@@ -5227,7 +5209,8 @@ impl StorageAPI for SetDisks {
uploads.push(MultipartInfo {
bucket: bucket.to_owned(),
object: object.to_owned(),
upload_id: base64_encode(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes()),
upload_id: base64_simd::URL_SAFE_NO_PAD
.encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes()),
initiated: Some(start_time),
..Default::default()
});
@@ -5348,6 +5331,14 @@ impl StorageAPI for SetDisks {
}
}
if let Some(checksum) = &opts.want_checksum {
user_defined.insert(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM.to_string(), checksum.checksum_type.to_string());
user_defined.insert(
rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE.to_string(),
checksum.checksum_type.obj_type().to_string(),
);
}
let (shuffle_disks, mut parts_metadatas) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
let mod_time = opts.mod_time.unwrap_or(OffsetDateTime::now_utc());
@@ -5362,7 +5353,8 @@ impl StorageAPI for SetDisks {
let upload_uuid = format!("{}x{}", Uuid::new_v4(), mod_time.unix_timestamp_nanos());
let upload_id = base64_encode(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_uuid).as_bytes());
let upload_id = base64_simd::URL_SAFE_NO_PAD
.encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_uuid).as_bytes());
let upload_path = Self::get_upload_id_dir(bucket, object, upload_uuid.as_str());
@@ -5379,7 +5371,11 @@ impl StorageAPI for SetDisks {
// evalDisks
Ok(MultipartUploadResult { upload_id })
Ok(MultipartUploadResult {
upload_id,
checksum_algo: user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM).cloned(),
checksum_type: user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE).cloned(),
})
}
#[tracing::instrument(skip(self))]
@@ -5467,6 +5463,25 @@ impl StorageAPI for SetDisks {
return Err(Error::other("part result number err"));
}
if let Some(cs) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM) {
let Some(checksum_type) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE) else {
return Err(Error::other("checksum type not found"));
};
if opts.want_checksum.is_some()
&& !opts.want_checksum.as_ref().is_some_and(|v| {
v.checksum_type
.is(rustfs_rio::ChecksumType::from_string_with_obj_type(cs, checksum_type))
})
{
return Err(Error::other(format!(
"checksum type mismatch, got {:?}, want {:?}",
opts.want_checksum.as_ref().unwrap(),
rustfs_rio::ChecksumType::from_string_with_obj_type(cs, checksum_type)
)));
}
}
for (i, part) in object_parts.iter().enumerate() {
if let Some(err) = &part.error {
error!("complete_multipart_upload part error: {:?}", &err);
@@ -5487,6 +5502,7 @@ impl StorageAPI for SetDisks {
part.mod_time,
part.actual_size,
part.index.clone(),
part.checksums.clone(),
);
}
@@ -6422,10 +6438,20 @@ mod tests {
CompletePart {
part_num: 1,
etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()),
checksum_crc32: None,
checksum_crc32c: None,
checksum_sha1: None,
checksum_sha256: None,
checksum_crc64nvme: None,
},
CompletePart {
part_num: 2,
etag: Some("098f6bcd4621d373cade4e832627b4f6".to_string()),
checksum_crc32: None,
checksum_crc32c: None,
checksum_sha1: None,
checksum_sha256: None,
checksum_crc64nvme: None,
},
];
@@ -6442,6 +6468,11 @@ mod tests {
let single_part = vec![CompletePart {
part_num: 1,
etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()),
checksum_crc32: None,
checksum_crc32c: None,
checksum_sha1: None,
checksum_sha256: None,
checksum_crc64nvme: None,
}];
let single_result = get_complete_multipart_md5(&single_part);
assert!(single_result.ends_with("-1"));