fix(s3): improve GitLab registry compatibility (#2596)

Co-authored-by: houseme <housemecn@gmail.com>

crates/ecstore/src/set_disk.rs

@@ -2808,18 +2808,10 @@ impl MultipartOperations for SetDisks {
         if part_numbers.is_empty() {
             return Ok(ret);
         }
-        let start_op = part_numbers.iter().find(|&&v| v != 0 && v == part_number_marker);
-        if part_number_marker > 0 && start_op.is_none() {
+        let Some(remaining_part_numbers) = parts_after_marker(&part_numbers, part_number_marker) else {
             return Ok(ret);
-        }
-
-        if let Some(start) = start_op {
-            if start + 1 > part_numbers.len() {
-                return Ok(ret);
-            }
-
-            part_numbers = part_numbers[start + 1..].to_vec();
-        }
+        };
+        part_numbers = remaining_part_numbers.to_vec();
 
         let mut parts = Vec::with_capacity(part_numbers.len());
 
@@ -3346,22 +3338,17 @@ impl MultipartOperations for SetDisks {
                     return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
                 };
 
-                let part_crc = match checksum_type {
-                    rustfs_rio::ChecksumType::SHA256 => p.checksum_sha256.clone(),
-                    rustfs_rio::ChecksumType::SHA1 => p.checksum_sha1.clone(),
-                    rustfs_rio::ChecksumType::CRC32 => p.checksum_crc32.clone(),
-                    rustfs_rio::ChecksumType::CRC32C => p.checksum_crc32c.clone(),
-                    rustfs_rio::ChecksumType::CRC64_NVME => p.checksum_crc64nvme.clone(),
-                    _ => {
-                        error!(
-                            "complete_multipart_upload checksum type={checksum_type}, part_id={}, bucket={}, object={}",
-                            p.part_num, bucket, object
-                        );
-                        return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
-                    }
+                let Some(part_crc) = complete_part_checksum(p, checksum_type) else {
+                    error!(
+                        "complete_multipart_upload checksum type={checksum_type}, part_id={}, bucket={}, object={}",
+                        p.part_num, bucket, object
+                    );
+                    return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
                 };
 
-                if part_crc.clone().unwrap_or_default() != crc {
+                if let Some(part_crc) = part_crc
+                    && part_crc != crc
+                {
                     error!("complete_multipart_upload checksum_type={checksum_type:?}, part_crc={part_crc:?}, crc={crc:?}");
                     error!(
                         "complete_multipart_upload checksum mismatch part_id={}, bucket={}, object={}",
@@ -4283,6 +4270,28 @@ fn get_complete_multipart_md5(parts: &[CompletePart]) -> String {
     format!("{}-{}", etag_hex, parts.len())
 }
 
+fn complete_part_checksum(part: &CompletePart, checksum_type: rustfs_rio::ChecksumType) -> Option<Option<String>> {
+    match checksum_type.base() {
+        rustfs_rio::ChecksumType::SHA256 => Some(part.checksum_sha256.clone()),
+        rustfs_rio::ChecksumType::SHA1 => Some(part.checksum_sha1.clone()),
+        rustfs_rio::ChecksumType::CRC32 => Some(part.checksum_crc32.clone()),
+        rustfs_rio::ChecksumType::CRC32C => Some(part.checksum_crc32c.clone()),
+        rustfs_rio::ChecksumType::CRC64_NVME => Some(part.checksum_crc64nvme.clone()),
+        _ => None,
+    }
+}
+
+fn parts_after_marker(part_numbers: &[usize], part_number_marker: usize) -> Option<&[usize]> {
+    if part_number_marker == 0 {
+        return Some(part_numbers);
+    }
+
+    part_numbers
+        .iter()
+        .position(|&part_number| part_number != 0 && part_number == part_number_marker)
+        .map(|index| &part_numbers[index + 1..])
+}
+
 pub fn canonicalize_etag(etag: &str) -> String {
     let re = Regex::new("\"*?([^\"]*?)\"*?$").unwrap();
     re.replace_all(etag, "$1").to_string()
@@ -5547,6 +5556,39 @@ mod tests {
         assert!(!is_valid_storage_class("standard")); // lowercase
     }
 
+    #[test]
+    fn complete_part_checksum_accepts_missing_value_and_uses_base_type() {
+        let missing_checksum_part = CompletePart::default();
+        assert_eq!(
+            complete_part_checksum(&missing_checksum_part, rustfs_rio::ChecksumType::CRC64_NVME),
+            Some(None)
+        );
+
+        let full_object_crc32 =
+            rustfs_rio::ChecksumType(rustfs_rio::ChecksumType::CRC32.0 | rustfs_rio::ChecksumType::FULL_OBJECT.0);
+        let part = CompletePart {
+            checksum_crc32: Some("AAAAAA==".to_string()),
+            ..Default::default()
+        };
+        assert_eq!(complete_part_checksum(&part, full_object_crc32), Some(Some("AAAAAA==".to_string())));
+    }
+
+    #[test]
+    fn parts_after_marker_uses_marker_position() {
+        let part_numbers = (1..=1002).collect::<Vec<_>>();
+
+        let remaining = parts_after_marker(&part_numbers, 1000).expect("marker should exist");
+
+        assert_eq!(remaining, &[1001, 1002]);
+    }
+
+    #[test]
+    fn parts_after_marker_returns_none_for_missing_marker() {
+        let part_numbers = vec![1, 2, 3];
+
+        assert!(parts_after_marker(&part_numbers, 4).is_none());
+    }
+
     #[test]
     fn test_is_cold_storage_class() {
         // Test cold storage classes

crates/rio/src/hash_reader.rs

@@ -408,6 +408,24 @@ impl HashReader {
         Ok(())
     }
 
+    pub fn add_calculated_checksum(&mut self, checksum_type: ChecksumType) -> Result<(), std::io::Error> {
+        if !checksum_type.is_set() {
+            return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid checksum type"));
+        }
+
+        let Some(hasher) = checksum_type.hasher() else {
+            return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid checksum type"));
+        };
+
+        self.content_hash = Some(Checksum {
+            checksum_type,
+            ..Default::default()
+        });
+        self.content_hasher = Some(hasher);
+
+        Ok(())
+    }
+
     pub fn checksum(&self) -> Option<Checksum> {
         if self
             .content_hash
@@ -569,7 +587,13 @@ impl AsyncRead for HashReader {
 
                         let content_hash = hasher.finalize();
 
-                        if content_hash != expected_content_hash.raw {
+                        if expected_content_hash.raw.is_empty()
+                            && expected_content_hash.encoded.is_empty()
+                            && !expected_content_hash.checksum_type.trailing()
+                        {
+                            expected_content_hash.raw = content_hash;
+                            expected_content_hash.encoded = general_purpose::STANDARD.encode(&expected_content_hash.raw);
+                        } else if content_hash != expected_content_hash.raw {
                             let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower);
                             let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower);
                             error!(
@@ -742,6 +766,25 @@ mod tests {
         assert_eq!(buf, data);
     }
 
+    #[tokio::test]
+    async fn test_add_calculated_checksum_records_checksum() {
+        let data = b"server-side copy checksum";
+        let reader = BufReader::new(Cursor::new(&data[..]));
+        let mut hash_reader = HashReader::from_stream(reader, data.len() as i64, data.len() as i64, None, None, false).unwrap();
+
+        hash_reader.add_calculated_checksum(ChecksumType::CRC64_NVME).unwrap();
+
+        let mut buf = Vec::new();
+        hash_reader.read_to_end(&mut buf).await.unwrap();
+
+        let expected = Checksum::new_from_data(ChecksumType::CRC64_NVME, data).unwrap();
+        let checksums = hash_reader.content_crc();
+
+        assert_eq!(buf, data);
+        assert_eq!(hash_reader.content_crc_type(), Some(ChecksumType::CRC64_NVME));
+        assert_eq!(checksums.get("CRC64NVME"), Some(&expected.encoded));
+    }
+
     #[tokio::test]
     async fn test_hashreader_new_logic() {
         let data = b"test data";

crates/scanner/tests/lifecycle_integration_test.rs

@@ -1225,10 +1225,11 @@ mod serial_tests {
         let object_name = "test/object.txt";
 
         create_test_bucket(&ecstore, bucket_name.as_str()).await;
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
+
         set_bucket_lifecycle(bucket_name.as_str())
             .await
             .expect("Failed to set lifecycle configuration");
-        upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
 
         assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);