gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 06:13:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(s3): complete CopyObject checksum support (#5178)

Browse Source
This commit is contained in:
cxymds
2026-07-24 16:57:11 +08:00
committed by GitHub
parent 3132637294
commit 9eaf5fc8e3
8 changed files with 495 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/app/multipart_usecase.rs
+3 -2
View File
@@ -554,11 +554,11 @@ impl DefaultMultipartUsecase {
// checksum: stored (decrypted) values take precedence over the request input;
// additional algorithms (XXHash3/64/128, SHA-512, MD5), which have no typed
// CompleteMultipartUploadOutput field, are echoed as raw response headers (#1261).
let (checksums, _is_multipart) = obj_info
let (checksums, is_multipart) = obj_info
.decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers)
.map_err(ApiError::from)?;
let classified = crate::app::object_usecase::classify_response_checksums(checksums);
let classified = crate::app::object_usecase::classify_response_checksums(checksums, is_multipart);
let checksum_crc32 = classified.crc32.or(input.checksum_crc32);
let checksum_crc32c = classified.crc32c.or(input.checksum_crc32c);
let checksum_sha1 = classified.sha1.or(input.checksum_sha1);
@@ -1352,6 +1352,7 @@ impl DefaultMultipartUsecase {
checksum_crc64nvme: checksum_value(rustfs_rio::ChecksumType::CRC64_NVME),
e_tag: part_info.etag.map(|etag| to_s3s_etag(&etag)),
last_modified: part_info.last_mod.map(Timestamp::from),
..Default::default()
};
let output = UploadPartCopyOutput {
rustfs/src/app/object_usecase.rs
+77 -47
View File
@@ -2368,11 +2368,8 @@ fn apply_trailing_checksums(
}
/// Checksums resolved from stored (decrypted) metadata for a response. The five
/// s3s-typed algorithms fill named fields; the AWS 2026-04 additional algorithms
/// (XXHash3/64/128, SHA-512, MD5), which s3s has no typed `*Output` field for, land
/// in `extra` and are emitted as raw response headers via
/// [`inject_additional_checksum_headers`]. Built by [`classify_response_checksums`]
/// so the typed/extra split lives in exactly one place.
/// legacy algorithms fill named fields; the additional algorithms land in `extra`
/// for raw-header response paths and DTOs that expose their newer typed fields.
#[derive(Default)]
pub(crate) struct ResponseChecksums {
pub(crate) crc32: Option<String>,
@@ -2384,11 +2381,11 @@ pub(crate) struct ResponseChecksums {
pub(crate) extra: Vec<(&'static str, String)>,
}
/// Split decrypted checksum (key, value) pairs into the five s3s-typed fields and the
/// additional-algorithm `extra` headers. Single source of truth for every response
/// Split decrypted checksum pairs into the five legacy fields and the additional
/// algorithm values. Single source of truth for every response
/// path (GetObject / HeadObject / GetObjectAttributes / CompleteMultipartUpload),
/// replacing what used to be five copies of this match loop.
pub(crate) fn classify_response_checksums<I>(pairs: I) -> ResponseChecksums
pub(crate) fn classify_response_checksums<I>(pairs: I, is_multipart: bool) -> ResponseChecksums
where
I: IntoIterator<Item = (String, String)>,
{
@@ -2412,6 +2409,9 @@ where
}
}
}
if is_multipart && c.checksum_type.is_none() {
c.checksum_type = Some(ChecksumType::from("COMPOSITE".to_string()));
}
c
}
@@ -4109,13 +4109,12 @@ impl DefaultObjectUsecase {
&& checksum_mode.to_str().unwrap_or_default() == "ENABLED"
&& rs.is_none()
{
let (decrypted_checksums, _is_multipart) =
info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| {
error!(error = %e, "GetObject checksum decryption failed");
ApiError::from(e)
})?;
let (decrypted_checksums, is_multipart) = info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| {
error!(error = %e, "GetObject checksum decryption failed");
ApiError::from(e)
})?;
return Ok(classify_response_checksums(decrypted_checksums));
return Ok(classify_response_checksums(decrypted_checksums, is_multipart));
}
Ok(ResponseChecksums::default())
@@ -5626,7 +5625,7 @@ impl DefaultObjectUsecase {
};
let checksum = if requested(ObjectAttributes::CHECKSUM) {
let (checksums, _is_multipart) = info.decrypt_checksums(0, &req.headers).map_err(ApiError::from)?;
let (checksums, is_multipart) = info.decrypt_checksums(0, &req.headers).map_err(ApiError::from)?;
// GetObjectAttributes returns checksums in the XML body, and s3s's Checksum
// type has no field for the additional algorithms, so `extra` cannot be
// surfaced here (unlike the header-based GET/HEAD paths) — an s3s limitation
@@ -5639,7 +5638,7 @@ impl DefaultObjectUsecase {
crc64nvme: checksum_crc64nvme,
checksum_type,
..
} = classify_response_checksums(checksums);
} = classify_response_checksums(checksums, is_multipart);
Some(Checksum {
checksum_crc32,
@@ -5648,6 +5647,7 @@ impl DefaultObjectUsecase {
checksum_sha256,
checksum_crc64nvme,
checksum_type,
..Default::default()
})
} else {
None
@@ -5674,7 +5674,7 @@ impl DefaultObjectUsecase {
let is_truncated = end < info.parts.len();
for part in &info.parts[start_at..end] {
let (checksums, _is_multipart) = info.decrypt_checksums(part.number, &req.headers).map_err(ApiError::from)?;
let (checksums, is_multipart) = info.decrypt_checksums(part.number, &req.headers).map_err(ApiError::from)?;
// Additional algorithms cannot be surfaced in the ObjectPart XML body
// (s3s has no field); same limitation as the object-level attributes above.
let ResponseChecksums {
@@ -5684,7 +5684,7 @@ impl DefaultObjectUsecase {
sha256: checksum_sha256,
crc64nvme: checksum_crc64nvme,
..
} = classify_response_checksums(checksums);
} = classify_response_checksums(checksums, is_multipart);
let part_size = if part.actual_size > 0 {
part.actual_size
@@ -5702,6 +5702,7 @@ impl DefaultObjectUsecase {
checksum_crc64nvme,
part_number: i32::try_from(part.number).ok(),
size: Some(part_size),
..Default::default()
});
}
@@ -5793,6 +5794,12 @@ impl DefaultObjectUsecase {
checksum_algorithm,
..
} = req.input.clone();
let requested_checksum_type = checksum_algorithm
.as_ref()
.map(|algorithm| rustfs_rio::ChecksumType::from_string(algorithm.as_str()));
if requested_checksum_type.is_some_and(|checksum_type| !checksum_type.is_set()) {
return Err(s3_error!(InvalidArgument, "Unsupported checksum algorithm"));
}
let (src_bucket, src_key, version_id) = match copy_source {
CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)),
CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)),
@@ -5996,12 +6003,9 @@ impl DefaultObjectUsecase {
// rather than re-hashing every byte.
let src_checksum = src_info.checksum.as_ref().and_then(|bytes| {
let (pairs, _) = rustfs_rio::read_checksums(bytes.as_ref(), 0);
pairs.into_iter().find_map(|(k, v)| {
rustfs_rio::ChecksumType::from_string(&k)
.is_s3s_typed()
.then(|| rustfs_rio::Checksum::new_from_string(&k, &v))
.flatten()
})
pairs
.into_iter()
.find_map(|(k, v)| rustfs_rio::Checksum::new_from_string(&k, &v))
});
// Validate copy source conditions
@@ -6125,12 +6129,9 @@ impl DefaultObjectUsecase {
// none is requested, carry the source object's stored checksum over unchanged — the copy
// does not alter the plaintext, so re-hashing would be wasted work and would flatten a
// multipart composite value.
match checksum_algorithm.as_ref() {
Some(algo) => {
let ct = rustfs_rio::ChecksumType::from_string(algo.as_str());
if ct.is_set() {
reader.add_calculated_checksum(ct).map_err(ApiError::from)?;
}
match requested_checksum_type {
Some(checksum_type) => {
reader.add_calculated_checksum(checksum_type).map_err(ApiError::from)?;
}
None => {
if let Some(cs) = src_checksum {
@@ -6221,11 +6222,26 @@ impl DefaultObjectUsecase {
// / HeadObject do so the value is identical to a later checksum-mode HEAD/GET (#4996).
let response_checksums = oi
.decrypt_checksums(0, &req.headers)
.map(|(pairs, _)| classify_response_checksums(pairs))
.map(|(pairs, is_multipart)| classify_response_checksums(pairs, is_multipart))
.unwrap_or_default();
// warn!("copy_object oi {:?}", &oi);
let object_info = oi.clone();
let mut checksum_md5 = None;
let mut checksum_sha512 = None;
let mut checksum_xxhash3 = None;
let mut checksum_xxhash64 = None;
let mut checksum_xxhash128 = None;
for (name, value) in response_checksums.extra {
match name {
"x-amz-checksum-md5" => checksum_md5 = Some(value),
"x-amz-checksum-sha512" => checksum_sha512 = Some(value),
"x-amz-checksum-xxhash3" => checksum_xxhash3 = Some(value),
"x-amz-checksum-xxhash64" => checksum_xxhash64 = Some(value),
"x-amz-checksum-xxhash128" => checksum_xxhash128 = Some(value),
_ => {}
}
}
let copy_object_result = CopyObjectResult {
e_tag: oi.etag.map(|etag| to_s3s_etag(&etag)),
last_modified: oi.mod_time.map(Timestamp::from),
@@ -6234,6 +6250,11 @@ impl DefaultObjectUsecase {
checksum_sha1: response_checksums.sha1,
checksum_sha256: response_checksums.sha256,
checksum_crc64nvme: response_checksums.crc64nvme,
checksum_md5,
checksum_sha512,
checksum_xxhash3,
checksum_xxhash64,
checksum_xxhash128,
checksum_type: response_checksums.checksum_type,
};
@@ -7122,10 +7143,10 @@ impl DefaultObjectUsecase {
&& checksum_mode.to_str().unwrap_or_default() == "ENABLED"
&& rs.is_none()
{
let (checksums, _is_multipart) = info
let (checksums, is_multipart) = info
.decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers)
.map_err(ApiError::from)?;
classify_response_checksums(checksums)
classify_response_checksums(checksums, is_multipart)
} else {
ResponseChecksums::default()
};
@@ -8109,24 +8130,30 @@ mod tests {
#[test]
fn classify_response_checksums_splits_typed_and_extra() {
// Typed algorithms fill named fields; nothing spills into extra.
let c = classify_response_checksums(vec![
("CRC32".to_string(), "AAAAAA==".to_string()),
("SHA256".to_string(), "c2hhMjU2".to_string()),
("CRC64NVME".to_string(), "Zm9vYmFyCg==".to_string()),
]);
let c = classify_response_checksums(
vec![
("CRC32".to_string(), "AAAAAA==".to_string()),
("SHA256".to_string(), "c2hhMjU2".to_string()),
("CRC64NVME".to_string(), "Zm9vYmFyCg==".to_string()),
],
false,
);
assert_eq!(c.crc32.as_deref(), Some("AAAAAA=="));
assert_eq!(c.sha256.as_deref(), Some("c2hhMjU2"));
assert_eq!(c.crc64nvme.as_deref(), Some("Zm9vYmFyCg=="));
assert!(c.extra.is_empty(), "typed algorithms must not land in extra");
// Additional algorithms land in extra keyed by their response-header name.
let c = classify_response_checksums(vec![
("XXHASH3".to_string(), "eHhoMw==".to_string()),
("XXHASH64".to_string(), "eHhoNjQ=".to_string()),
("XXHASH128".to_string(), "eHhoMTI4".to_string()),
("SHA512".to_string(), "c2hhNTEy".to_string()),
("MD5".to_string(), "bWQ1".to_string()),
]);
let c = classify_response_checksums(
vec![
("XXHASH3".to_string(), "eHhoMw==".to_string()),
("XXHASH64".to_string(), "eHhoNjQ=".to_string()),
("XXHASH128".to_string(), "eHhoMTI4".to_string()),
("SHA512".to_string(), "c2hhNTEy".to_string()),
("MD5".to_string(), "bWQ1".to_string()),
],
false,
);
assert!(c.crc32.is_none() && c.sha256.is_none() && c.crc64nvme.is_none());
let names: Vec<&str> = c.extra.iter().map(|(n, _)| *n).collect();
for expected in [
@@ -8141,12 +8168,15 @@ mod tests {
assert_eq!(c.extra.len(), 5);
// The checksum-type marker is captured as the type, not mistaken for an algorithm.
let c = classify_response_checksums(vec![(AMZ_CHECKSUM_TYPE.to_string(), "COMPOSITE".to_string())]);
let c = classify_response_checksums(vec![(AMZ_CHECKSUM_TYPE.to_string(), "COMPOSITE".to_string())], false);
assert!(c.checksum_type.is_some());
assert!(c.extra.is_empty() && c.crc32.is_none());
let c = classify_response_checksums(vec![("CRC32".to_string(), "AAAAAA==-2".to_string())], true);
assert_eq!(c.checksum_type.as_ref().map(ChecksumType::as_str), Some("COMPOSITE"));
// Empty input yields an all-default result.
let c = classify_response_checksums(Vec::<(String, String)>::new());
let c = classify_response_checksums(Vec::<(String, String)>::new(), false);
assert!(c.crc32.is_none() && c.extra.is_empty() && c.checksum_type.is_none());
}