fix(s3): improve GitLab registry compatibility (#2596)

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
安正超
2026-04-19 10:10:42 +08:00
committed by GitHub
parent 93d0606cbd
commit 457f4e0170
7 changed files with 292 additions and 39 deletions
+67 -25
View File
@@ -2808,18 +2808,10 @@ impl MultipartOperations for SetDisks {
if part_numbers.is_empty() {
return Ok(ret);
}
let start_op = part_numbers.iter().find(|&&v| v != 0 && v == part_number_marker);
if part_number_marker > 0 && start_op.is_none() {
let Some(remaining_part_numbers) = parts_after_marker(&part_numbers, part_number_marker) else {
return Ok(ret);
}
if let Some(start) = start_op {
if start + 1 > part_numbers.len() {
return Ok(ret);
}
part_numbers = part_numbers[start + 1..].to_vec();
}
};
part_numbers = remaining_part_numbers.to_vec();
let mut parts = Vec::with_capacity(part_numbers.len());
@@ -3346,22 +3338,17 @@ impl MultipartOperations for SetDisks {
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
};
let part_crc = match checksum_type {
rustfs_rio::ChecksumType::SHA256 => p.checksum_sha256.clone(),
rustfs_rio::ChecksumType::SHA1 => p.checksum_sha1.clone(),
rustfs_rio::ChecksumType::CRC32 => p.checksum_crc32.clone(),
rustfs_rio::ChecksumType::CRC32C => p.checksum_crc32c.clone(),
rustfs_rio::ChecksumType::CRC64_NVME => p.checksum_crc64nvme.clone(),
_ => {
error!(
"complete_multipart_upload checksum type={checksum_type}, part_id={}, bucket={}, object={}",
p.part_num, bucket, object
);
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
}
let Some(part_crc) = complete_part_checksum(p, checksum_type) else {
error!(
"complete_multipart_upload checksum type={checksum_type}, part_id={}, bucket={}, object={}",
p.part_num, bucket, object
);
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
};
if part_crc.clone().unwrap_or_default() != crc {
if let Some(part_crc) = part_crc
&& part_crc != crc
{
error!("complete_multipart_upload checksum_type={checksum_type:?}, part_crc={part_crc:?}, crc={crc:?}");
error!(
"complete_multipart_upload checksum mismatch part_id={}, bucket={}, object={}",
@@ -4283,6 +4270,28 @@ fn get_complete_multipart_md5(parts: &[CompletePart]) -> String {
format!("{}-{}", etag_hex, parts.len())
}
fn complete_part_checksum(part: &CompletePart, checksum_type: rustfs_rio::ChecksumType) -> Option<Option<String>> {
match checksum_type.base() {
rustfs_rio::ChecksumType::SHA256 => Some(part.checksum_sha256.clone()),
rustfs_rio::ChecksumType::SHA1 => Some(part.checksum_sha1.clone()),
rustfs_rio::ChecksumType::CRC32 => Some(part.checksum_crc32.clone()),
rustfs_rio::ChecksumType::CRC32C => Some(part.checksum_crc32c.clone()),
rustfs_rio::ChecksumType::CRC64_NVME => Some(part.checksum_crc64nvme.clone()),
_ => None,
}
}
fn parts_after_marker(part_numbers: &[usize], part_number_marker: usize) -> Option<&[usize]> {
if part_number_marker == 0 {
return Some(part_numbers);
}
part_numbers
.iter()
.position(|&part_number| part_number != 0 && part_number == part_number_marker)
.map(|index| &part_numbers[index + 1..])
}
pub fn canonicalize_etag(etag: &str) -> String {
let re = Regex::new("\"*?([^\"]*?)\"*?$").unwrap();
re.replace_all(etag, "$1").to_string()
@@ -5547,6 +5556,39 @@ mod tests {
assert!(!is_valid_storage_class("standard")); // lowercase
}
#[test]
fn complete_part_checksum_accepts_missing_value_and_uses_base_type() {
let missing_checksum_part = CompletePart::default();
assert_eq!(
complete_part_checksum(&missing_checksum_part, rustfs_rio::ChecksumType::CRC64_NVME),
Some(None)
);
let full_object_crc32 =
rustfs_rio::ChecksumType(rustfs_rio::ChecksumType::CRC32.0 | rustfs_rio::ChecksumType::FULL_OBJECT.0);
let part = CompletePart {
checksum_crc32: Some("AAAAAA==".to_string()),
..Default::default()
};
assert_eq!(complete_part_checksum(&part, full_object_crc32), Some(Some("AAAAAA==".to_string())));
}
#[test]
fn parts_after_marker_uses_marker_position() {
let part_numbers = (1..=1002).collect::<Vec<_>>();
let remaining = parts_after_marker(&part_numbers, 1000).expect("marker should exist");
assert_eq!(remaining, &[1001, 1002]);
}
#[test]
fn parts_after_marker_returns_none_for_missing_marker() {
let part_numbers = vec![1, 2, 3];
assert!(parts_after_marker(&part_numbers, 4).is_none());
}
#[test]
fn test_is_cold_storage_class() {
// Test cold storage classes