test(ecstore): add validation suite coverage gates (#4378)

2026-08-22 12:26:37 +00:00 · 2026-07-07 23:50:12 +08:00
parent 3ed414cdb4
commit 742a59884d
7 changed files with 1634 additions and 63 deletions
@@ -3427,6 +3427,43 @@ impl SetDisks {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use std::io::Cursor;
+    use tokio::io::AsyncReadExt;
+
+    fn metadata_test_fileinfo(object: &str) -> FileInfo {
+        let mut fi = FileInfo::new(object, 2, 2);
+        fi.volume = "bucket".to_string();
+        fi.name = object.to_string();
+        fi.size = 1;
+        fi.erasure.index = 1;
+        fi.metadata.insert("etag".to_string(), "etag-1".to_string());
+        fi.add_object_part(1, "part-etag-1".to_string(), 1, None, 1, None, None);
+        fi
+    }
+
+    fn read_part_test_part(number: usize, etag: &str) -> ObjectPartInfo {
+        ObjectPartInfo {
+            number,
+            etag: etag.to_string(),
+            ..Default::default()
+        }
+    }
+
+    fn read_part_test_error(number: usize, error: &str) -> ObjectPartInfo {
+        ObjectPartInfo {
+            number,
+            error: Some(error.to_string()),
+            ..Default::default()
+        }
+    }
+
+    fn failed_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
+        Box::pin(async { ReadRepairAdmissionOutcome::Failed("injected submit failure".to_string()) })
+    }
+
+    fn accepted_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
+        Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) })
+    }

    #[test]
    fn dangling_delete_grace_defaults_to_one_hour() {
@@ -3444,4 +3481,224 @@ mod tests {
            assert!(dangling_delete_grace().is_zero());
        });
    }
+
+    #[tokio::test]
+    async fn multipart_codec_streaming_reader_zero_buffer_is_noop() {
+        let reader = tokio::io::BufReader::new(Cursor::new(b"payload".to_vec()));
+        let mut reader = MultipartCodecStreamingReader::new(vec![Box::new(reader)]);
+        let mut out = [];
+
+        let read = reader
+            .read(&mut out)
+            .await
+            .expect("zero-length read should not poll inner readers");
+
+        assert_eq!(read, 0);
+        assert_eq!(reader.readers.len(), 1);
+    }
+
+    #[test]
+    fn metadata_fanout_observation_classifies_invalid_and_ignored_results() {
+        let invalid = MetadataFanoutObservation::from_file_info(&FileInfo::default(), Duration::from_millis(7));
+        assert_eq!(invalid.outcome, GET_METADATA_RESPONSE_ERROR);
+        assert!(!invalid.valid);
+        assert!(!invalid.ignored);
+
+        let ignored = MetadataFanoutObservation::from_error(&DiskError::DiskNotFound, Duration::from_millis(9));
+        assert_eq!(ignored.outcome, GET_METADATA_RESPONSE_DISK_NOT_FOUND);
+        assert!(!ignored.valid);
+        assert!(ignored.ignored);
+
+        let corrupt = MetadataFanoutObservation::from_error(&DiskError::FileCorrupt, Duration::from_millis(11));
+        assert_eq!(corrupt.outcome, GET_METADATA_RESPONSE_CORRUPT);
+        assert!(!corrupt.ignored);
+    }
+
+    #[test]
+    fn metadata_fanout_diagnostics_reports_counts_and_latency_edges() {
+        let diagnostics = MetadataFanoutDiagnostics::new(
+            Duration::from_millis(40),
+            vec![
+                MetadataFanoutObservation {
+                    outcome: GET_METADATA_RESPONSE_VALID,
+                    elapsed: Duration::from_millis(30),
+                    valid: true,
+                    ignored: false,
+                },
+                MetadataFanoutObservation {
+                    outcome: GET_METADATA_RESPONSE_IGNORED,
+                    elapsed: Duration::from_millis(10),
+                    valid: false,
+                    ignored: true,
+                },
+                MetadataFanoutObservation {
+                    outcome: GET_METADATA_RESPONSE_ERROR,
+                    elapsed: Duration::from_millis(20),
+                    valid: false,
+                    ignored: false,
+                },
+            ],
+        );
+
+        assert_eq!(diagnostics.total_responses(), 3);
+        assert_eq!(diagnostics.valid_responses(), 1);
+        assert_eq!(diagnostics.ignored_responses(), 1);
+        assert_eq!(diagnostics.error_responses(), 2);
+        assert_eq!(diagnostics.first_response_latency(), Some(Duration::from_millis(10)));
+        assert_eq!(diagnostics.first_valid_response_latency(), Some(Duration::from_millis(30)));
+        assert_eq!(diagnostics.slowest_response_latency(), Some(Duration::from_millis(30)));
+        assert_eq!(diagnostics.quorum_candidate_latency(0), Some(Duration::ZERO));
+        assert_eq!(diagnostics.quorum_candidate_latency(1), Some(Duration::from_millis(30)));
+        assert_eq!(diagnostics.quorum_candidate_latency(2), None);
+    }
+
+    #[test]
+    fn metadata_quorum_accumulator_counts_invalid_metadata_and_ignored_errors() {
+        let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true);
+
+        accumulator.observe_file_info(&FileInfo::default());
+        accumulator.observe_error(&DiskError::DiskNotFound);
+
+        assert_eq!(accumulator.hard_errors, 1);
+        assert_eq!(accumulator.ignored_errors, 1);
+        assert_eq!(accumulator.final_miss_reason(), GET_METADATA_EARLY_STOP_REASON_ERROR);
+    }
+
+    #[test]
+    fn metadata_quorum_accumulator_candidate_quorum_handles_zero_parity_and_invalid_candidates() {
+        let accumulator = MetadataQuorumAccumulator::new(4, 0, true);
+        let candidate = metadata_test_fileinfo("object");
+        assert_eq!(accumulator.candidate_read_quorum(&candidate), Some(4));
+        assert_eq!(accumulator.missing_response_quorum(), 4);
+
+        let accumulator = MetadataQuorumAccumulator::new(4, 2, true);
+        let mut deleted = candidate.clone();
+        deleted.deleted = true;
+        assert_eq!(accumulator.candidate_read_quorum(&deleted), None);
+
+        let mut empty = candidate.clone();
+        empty.size = 0;
+        assert_eq!(accumulator.candidate_read_quorum(&empty), None);
+
+        let mut impossible_parity = candidate;
+        impossible_parity.erasure.parity_blocks = 4;
+        assert_eq!(accumulator.candidate_read_quorum(&impossible_parity), None);
+    }
+
+    #[test]
+    fn confirmed_missing_part_error_recognizes_legacy_and_s3_markers() {
+        assert!(!is_confirmed_missing_part_error(None));
+        assert!(is_confirmed_missing_part_error(Some("file not found")));
+        assert!(is_confirmed_missing_part_error(Some("No such file or directory")));
+        assert!(is_confirmed_missing_part_error(Some("Specified part could not be found")));
+        assert!(is_confirmed_missing_part_error(Some("part.7 not found")));
+        assert!(!is_confirmed_missing_part_error(Some("part.7 missing")));
+        assert!(!is_confirmed_missing_part_error(Some("permission denied")));
+    }
+
+    #[test]
+    fn resolve_read_part_handles_mismatched_and_transient_responses_without_false_missing() {
+        let responses = vec![
+            Some(Vec::new()),
+            Some(vec![read_part_test_error(1, "permission denied")]),
+            None,
+        ];
+
+        let err = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
+            .expect_err("mismatched and transient responses must not be treated as confirmed missing");
+
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+    }
+
+    #[test]
+    fn resolve_read_part_accepts_alternate_missing_error_markers() {
+        let responses = vec![
+            Some(vec![read_part_test_error(1, "Specified part could not be found")]),
+            Some(vec![read_part_test_error(1, "part.1 not found")]),
+            Some(vec![read_part_test_part(1, "stale-etag")]),
+        ];
+
+        let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
+            .expect("alternate missing markers should satisfy missing quorum");
+
+        assert_eq!(part.number, 1);
+        assert_eq!(part.error.as_deref(), Some("part.1 not found"));
+        assert!(part.etag.is_empty());
+    }
+
+    #[test]
+    fn shard_read_costs_for_empty_disk_set_are_empty() {
+        assert!(shard_read_costs_for_disks(&[]).is_empty());
+    }
+
+    #[tokio::test]
+    async fn reserve_read_repair_heal_dedupes_by_object_version_and_set() {
+        let object = format!("object-{}", Uuid::new_v4());
+        let key = reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
+            .await
+            .expect("first read-repair reservation should be accepted");
+
+        assert_eq!(key.version_id.as_deref(), Some("version-1"));
+        assert!(
+            reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
+                .await
+                .is_none()
+        );
+
+        release_read_repair_heal_reservation(&key).await;
+        let retry_key = reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
+            .await
+            .expect("released read-repair reservation should allow retry");
+        release_read_repair_heal_reservation(&retry_key).await;
+    }
+
+    #[tokio::test]
+    async fn submit_read_repair_heal_releases_reservation_after_submitter_failure() {
+        let object = format!("object-{}", Uuid::new_v4());
+        submit_read_repair_heal_with_submitter(
+            ReadRepairHealSubmission {
+                bucket: "bucket",
+                object: &object,
+                version_id: None,
+                pool_index: 1,
+                set_index: 2,
+                part_number: Some(1),
+                reason: "test",
+            },
+            failed_read_repair_submitter,
+        )
+        .await;
+
+        for _ in 0..20 {
+            if let Some(key) = reserve_read_repair_heal("bucket", &object, None, 1, 2).await {
+                release_read_repair_heal_reservation(&key).await;
+                return;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+
+        panic!("failed read-repair submission should release its dedup reservation");
+    }
+
+    #[tokio::test]
+    async fn submit_read_repair_heal_keeps_admitted_reservation_deduped() {
+        let object = format!("object-{}", Uuid::new_v4());
+        submit_read_repair_heal_with_submitter(
+            ReadRepairHealSubmission {
+                bucket: "bucket",
+                object: &object,
+                version_id: None,
+                pool_index: 3,
+                set_index: 4,
+                part_number: None,
+                reason: "test",
+            },
+            accepted_read_repair_submitter,
+        )
+        .await;
+
+        assert!(reserve_read_repair_heal("bucket", &object, None, 3, 4).await.is_none());
+        let key = ReadRepairHealCacheKey::new("bucket", &object, None, 3, 4);
+        release_read_repair_heal_reservation(&key).await;
+    }
 }