feat(heal): disk-walk UNION enumeration to heal sub-quorum versions (backlog#920) (#4527)

B5 switched heal enumeration to list_object_versions, which only reflects the read-quorum metadata view: a version present on fewer than read-quorum disks was never enumerated, so it was never healed. Add a per-erasure-set disk-walk UNION enumerator (mirrors MinIO global-heal.go objQuorum=1 listPathRaw + mergeXLV2Versions) that surfaces every (object, version) present on ANY disk and feeds each to the existing per-version heal_object. - filemeta: MetaCacheEntries::resolve_union (dir_quorum=1/obj_quorum=1) yields the cross-disk version union at one tested seam. - ecstore: SetDisks::heal_walk_versions_page (list_path_raw fan-out, min_disks=1, dual object/version page bound, inclusive-forward de-overlap) + ECStore delegator + HealWalkVersion. - ecstore data-safety guard: before dangling-delete, try_regenerate_recoverable_meta physically probes part files via check_parts; when >= data_blocks data shards survive (meta lost but data recoverable) it regenerates xl.meta from a surviving FileInfo with the correct per-disk shard index instead of dangling-deleting. Genuine torn writes (< data_blocks) keep the current behavior — no resurrection. - heal: dw1: forward-marker cursor codec (reuses ResumeState.resume_cursor, idempotent restart on foreign tokens); list_versions_for_heal_page_disk_walk trait method (default falls back to the B5 read-quorum path); heal_bucket_with_resume selects the disk-walk enumerator when scan_mode==Deep || source==AutoHeal, else the unchanged B5 path; anti-loop guard aborts on (empty && truncated). Closes rustfs/backlog#920
2026-08-25 13:36:50 +00:00 · 2026-07-09 01:07:54 +08:00
parent 2055044cb4
commit 3531abb34a
12 changed files with 1515 additions and 6 deletions
@@ -4018,6 +4018,69 @@ mod tests {
        assert_eq!(merged, vec![valid]);
    }

+    /// Build a single distinct Object version header for the union tests.
+    fn union_version(version_u128: u128, mod_unix: i64) -> FileMetaShallowVersion {
+        FileMetaShallowVersion {
+            header: FileMetaVersionHeader {
+                version_id: Some(Uuid::from_u128(version_u128)),
+                mod_time: Some(OffsetDateTime::from_unix_timestamp(mod_unix).expect("valid timestamp")),
+                signature: [0x11, 0x22, 0x33, 0x44],
+                version_type: VersionType::Object,
+                flags: 0,
+                ec_n: 4,
+                ec_m: 2,
+            },
+            meta: Vec::new(),
+        }
+    }
+
+    /// backlog#920: quorum==1 (union) must surface EVERY version present on ANY
+    /// per-disk stream, even when the per-disk version sets are fully DISJOINT
+    /// (a version living on a single disk). This is the enumeration guarantee the
+    /// sub-quorum disk-walk depends on.
+    #[test]
+    fn merge_union_quorum1_yields_full_union_over_disjoint_disks() {
+        let a = union_version(0xA, 1_705_312_300);
+        let b = union_version(0xB, 1_705_312_200);
+        let c = union_version(0xC, 1_705_312_100);
+
+        // Three disks, each holding exactly ONE distinct version (disjoint sets).
+        let disks = [vec![a], vec![b], vec![c]];
+        let merged = merge_file_meta_versions(1, true, 0, &disks);
+
+        let ids: std::collections::HashSet<Option<Uuid>> = merged.iter().map(|v| v.header.version_id).collect();
+        assert_eq!(merged.len(), 3, "union must retain all three disjoint versions: {merged:?}");
+        assert!(ids.contains(&Some(Uuid::from_u128(0xA))));
+        assert!(ids.contains(&Some(Uuid::from_u128(0xB))));
+        assert!(ids.contains(&Some(Uuid::from_u128(0xC))));
+
+        // Contrast: read-quorum (2) over the same disjoint streams surfaces NONE,
+        // because no version reaches two agreeing disks. This is the exact gap.
+        let read_quorum = merge_file_meta_versions(2, true, 0, &disks);
+        assert!(
+            read_quorum.is_empty(),
+            "read-quorum merge must drop every sub-quorum version, got {read_quorum:?}"
+        );
+    }
+
+    /// The equal-modTime / distinct-versionId retain-and-advance branch: two
+    /// versions sharing a mod_time but with different ids must BOTH survive the
+    /// union merge (they are not collapsed as duplicates).
+    #[test]
+    fn merge_union_quorum1_retains_equal_modtime_distinct_version_ids() {
+        let same_time = 1_705_312_300;
+        let a = union_version(0xAA, same_time);
+        let b = union_version(0xBB, same_time);
+
+        let disks = [vec![a], vec![b]];
+        let merged = merge_file_meta_versions(1, true, 0, &disks);
+
+        let ids: std::collections::HashSet<Option<Uuid>> = merged.iter().map(|v| v.header.version_id).collect();
+        assert_eq!(merged.len(), 2, "equal-modTime distinct-id versions must both survive: {merged:?}");
+        assert!(ids.contains(&Some(Uuid::from_u128(0xAA))));
+        assert!(ids.contains(&Some(Uuid::from_u128(0xBB))));
+    }
+
    #[test]
    fn meta_object_init_free_version_rejects_invalid_tier_free_version_id() {
        let mut sys = HashMap::new();
@@ -343,6 +343,32 @@ impl MetaCacheEntries {
        self.resolve_inner(params, true)
    }

+    /// Resolve the cross-disk UNION of every version present on ANY disk slot.
+    ///
+    /// This mirrors MinIO's global heal enumeration (cmd/global-heal.go
+    /// `healErasureSet` -> `listPathRaw` with `objQuorum = 1` feeding
+    /// `mergeEntries`/`mergeXLV2Versions`): at quorum 1 the merge is forced
+    /// strict and returns the union of all per-disk version streams, so a version
+    /// that survives on FEWER than read-quorum disks is still surfaced for
+    /// healing. Read-quorum resolution (`resolve` with `obj_quorum >= 2`) would
+    /// silently drop such a sub-quorum version, which is exactly the durability
+    /// gap this enumerator closes (backlog#920).
+    ///
+    /// `bucket` is only used to tag the merged entry; `strict:false` lets the
+    /// non-strict header reconciliation collapse equal-but-differently-signed
+    /// replicas of the SAME version id while still retaining genuinely distinct
+    /// version ids.
+    pub fn resolve_union(&self, bucket: &str) -> Option<MetaCacheEntry> {
+        self.resolve(MetadataResolutionParams {
+            dir_quorum: 1,
+            obj_quorum: 1,
+            requested_versions: 0,
+            bucket: bucket.to_string(),
+            strict: false,
+            candidates: Vec::new(),
+        })
+    }
+
    fn resolve_inner(&self, mut params: MetadataResolutionParams, enforce_write_quorum: bool) -> Option<MetaCacheEntry> {
        if self.0.is_empty() {
            debug!(
@@ -1480,6 +1506,92 @@ mod tests {
        }
    }

+    /// Build an entry holding a single object version with an explicit version id
+    /// and mod_time, so a set of these can model DISJOINT per-disk version sets.
+    fn metacache_entry_single_version(version_u128: u128, mod_time: OffsetDateTime, etag: &str) -> MetaCacheEntry {
+        let mut metadata = HashMap::new();
+        metadata.insert("etag".to_string(), etag.to_string());
+
+        let mut fi = FileInfo::new("object", 4, 2);
+        fi.volume = "bucket".to_string();
+        fi.name = "object".to_string();
+        fi.version_id = Some(Uuid::from_u128(version_u128));
+        fi.versioned = true;
+        fi.size = 1;
+        fi.mod_time = Some(mod_time);
+        fi.metadata = metadata;
+
+        let mut meta = FileMeta::new();
+        meta.add_version(fi).expect("test file metadata should accept object version");
+        let encoded = meta.marshal_msg().expect("test file metadata should marshal");
+
+        MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: encoded,
+            cached: Some(meta),
+            reusable: false,
+        }
+    }
+
+    /// backlog#920: `resolve_union` surfaces a version present on a SINGLE slot
+    /// among four, while `resolve` at read-quorum (obj_quorum=2) drops it. This
+    /// documents the exact sub-quorum durability gap the disk-walk closes.
+    #[test]
+    fn resolve_union_surfaces_single_disk_version() {
+        let t0 = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let t1 = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+
+        // Slot 0 carries a UNIQUE version (0xBEEF) present nowhere else; the other
+        // three slots agree on a shared version (0xCAFE). Read-quorum sees only
+        // the shared one; union must see BOTH.
+        let unique = metacache_entry_single_version(0xBEEF, t1, "unique-etag");
+        let shared_a = metacache_entry_single_version(0xCAFE, t0, "shared-etag");
+        let shared_b = metacache_entry_single_version(0xCAFE, t0, "shared-etag");
+        let shared_c = metacache_entry_single_version(0xCAFE, t0, "shared-etag");
+
+        let entries = || {
+            MetaCacheEntries(vec![
+                Some(unique.clone()),
+                Some(shared_a.clone()),
+                Some(shared_b.clone()),
+                Some(shared_c.clone()),
+            ])
+        };
+
+        let union = entries().resolve_union("bucket").expect("union must resolve an entry");
+        let union_meta = union.cached.expect("union entry keeps cached metadata");
+        let union_ids: std::collections::HashSet<Option<Uuid>> =
+            union_meta.versions.iter().map(|v| v.header.version_id).collect();
+        assert!(
+            union_ids.contains(&Some(Uuid::from_u128(0xBEEF))),
+            "union must surface the single-slot version: {union_ids:?}"
+        );
+        assert!(
+            union_ids.contains(&Some(Uuid::from_u128(0xCAFE))),
+            "union must also keep the shared version: {union_ids:?}"
+        );
+
+        // Read-quorum resolution (obj_quorum=2) drops the single-slot version.
+        let read_quorum = entries()
+            .resolve(MetadataResolutionParams {
+                obj_quorum: 2,
+                dir_quorum: 2,
+                strict: false,
+                ..Default::default()
+            })
+            .expect("read-quorum must still resolve the shared version");
+        let rq_meta = read_quorum.cached.expect("read-quorum entry keeps cached metadata");
+        let rq_ids: std::collections::HashSet<Option<Uuid>> = rq_meta.versions.iter().map(|v| v.header.version_id).collect();
+        assert!(
+            !rq_ids.contains(&Some(Uuid::from_u128(0xBEEF))),
+            "read-quorum must DROP the single-slot version (the gap): {rq_ids:?}"
+        );
+        assert!(
+            rq_ids.contains(&Some(Uuid::from_u128(0xCAFE))),
+            "read-quorum must keep the shared version: {rq_ids:?}"
+        );
+    }
+
    #[test]
    fn resolve_rejects_partial_latest_and_returns_committed_previous_metadata() {
        let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");