fix(core-storage): fix critical correctness defects from core-storage reliability audit (#4222)

* fix(core-storage): fix critical correctness defects from core-storage audit

Fixes verified defects found in a deep audit of the core storage path
(erasure coding, disk persistence, quorum, heal, replication resync):

- ecstore/disk: rewrite live xl.meta atomically (temp+rename) in
  delete_versions_internal and write_metadata instead of in-place
  truncate, which exposed torn metadata to concurrent readers and
  crashes on the DeleteObjects hot path
- ecstore/erasure: allow heal to reconstruct from exactly data_shards
  bitrot-verified sources; requiring data_shards+1 made objects
  permanently unhealable after losing parity_shards disks
- ecstore/set_disk: direct-memory inline GET applied the erasure
  distribution permutation twice (shuffled inputs re-indexed through
  distribution), concatenating wrong shards into the response body in
  degraded reads; collect from canonical disk-ordered inputs
- ecstore/set_disk: heal now preserves the committed inline layout
  instead of recomputing it with a hardcoded unversioned threshold,
  which split quorum identity of healed replicas and caused endless
  re-heal churn
- ecstore/replication: resync results channel switched from
  broadcast(1) to mpsc; a lagged broadcast receiver ended the stats
  collector and every subsequent failure went uncounted, letting
  failed resyncs be marked completed
- ecstore/replication: ignore an empty persisted resync checkpoint;
  resuming with one skipped every object and marked the resync
  completed without replicating anything
- ecstore/replication: fix inverted not-found error classification in
  replicate_object/replicate_delete logging paths
- ecstore/erasure: guard decode paths against zero block_size or
  data_shards from corrupt on-disk metadata (divide-by-zero panic)
- ecstore/disk: os::read_dir no longer consumes the entry limit on
  entries it does not return (is_empty_dir misjudgment); create_file
  opens with O_TRUNC to avoid stale trailing bytes
- filemeta: treat Some(nil) version id as a null version in
  matches_not_strict; disk-loaded headers never store None, so the
  mod_time quorum guard for unversioned overwrites never fired and an
  interrupted overwrite could displace the committed version in merge
- filemeta: fix msgpack skip lengths for fixext (missed the ext type
  byte) and ext16/32 (over-skipped) unknown fields
- filemeta: return FileCorrupt instead of usize underflow when
  xl.meta is truncated inside the CRC trailer
- filemeta: surface delete-marker insertion failure in delete_version
  instead of reporting success when the data dir is shared

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(replication): drop duplicate cfg(test) etag import from boundary module

The test module already imports content_matches_by_etag locally, so the
top-level cfg(test) import is unused under -D warnings and fails clippy.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

crates/filemeta/src/filemeta.rs

@@ -665,14 +665,17 @@ impl FileMeta {
             err = self.add_version_filemata(ventry).err();
         }
 
-        if self.shared_data_dir_count(obj_version_id, obj_data_dir) > 0 {
-            return Ok(None);
-        }
-
+        // A failed delete-marker insertion must surface even when the data dir is
+        // shared: reporting success here silently turns the delete into a permanent
+        // delete that replication never propagates.
         if let Some(e) = err {
             return Err(e);
         }
 
+        if self.shared_data_dir_count(obj_version_id, obj_data_dir) > 0 {
+            return Ok(None);
+        }
+
         Ok(obj_data_dir)
     }
 

crates/filemeta/src/filemeta/msgp_decode.rs

@@ -179,28 +179,30 @@ pub(crate) fn skip_msgp_value<R: Read>(rd: &mut R) -> Result<()> {
             }
             return Ok(());
         }
-        Marker::FixExt1 => 1,
-        Marker::FixExt2 => 2,
-        Marker::FixExt4 => 4,
-        Marker::FixExt8 => 8,
-        Marker::FixExt16 => 16,
+        // fixext N = marker + 1 type byte + N data bytes
+        Marker::FixExt1 => 2,
+        Marker::FixExt2 => 3,
+        Marker::FixExt4 => 5,
+        Marker::FixExt8 => 9,
+        Marker::FixExt16 => 17,
+        // ext 8/16/32 = marker + length bytes (read here) + 1 type byte + data
         Marker::Ext8 => {
             let mut b = [0u8; 1];
             rd.read_exact(&mut b).map_err(Error::from)?;
             let len = b[0] as usize;
-            1 + len // type byte + data
+            1 + len
         }
         Marker::Ext16 => {
             let mut b = [0u8; 2];
             rd.read_exact(&mut b).map_err(Error::from)?;
             let len = u16::from_be_bytes(b) as usize;
-            2 + len // type bytes + data
+            1 + len
         }
         Marker::Ext32 => {
             let mut b = [0u8; 4];
             rd.read_exact(&mut b).map_err(Error::from)?;
             let len = u32::from_be_bytes(b) as usize;
-            4 + len // type bytes + data
+            1 + len
         }
         Marker::Reserved => 0,
     };

crates/filemeta/src/filemeta/version.rs

@@ -826,7 +826,10 @@ impl FileMetaVersionHeader {
 
     pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool {
         let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o);
-        if self.version_id.is_none() {
+        // Disk-loaded headers keep the null version as Some(nil), not None: all null
+        // versions share one id, so mod_time is the only thing distinguishing an
+        // interrupted overwrite from the committed version.
+        if self.version_id.is_none() || self.version_id == Some(Uuid::nil()) {
             ok = ok && self.mod_time == o.mod_time;
         }
 
@@ -2916,11 +2919,14 @@ pub async fn read_xl_meta_no_data<R: AsyncRead + Unpin>(reader: &mut R, size: us
                     return Err(Error::FileCorrupt);
                 }
 
-                let tmp = &buf[want..];
+                // The metadata block is followed by a 5-byte msgp uint32 CRC trailer;
+                // a file truncated inside the trailer is corrupt, not a shorter meta.
                 let crc_size = 5;
-                let other_size = tmp.len() - crc_size;
+                if buf.len() - want < crc_size {
+                    return Err(Error::FileCorrupt);
+                }
 
-                want += tmp.len() - other_size;
+                want += crc_size;
 
                 buf.truncate(want);
                 Ok(buf)