fix(core-storage): fix critical correctness defects from core-storage reliability audit (#4222)

* fix(core-storage): fix critical correctness defects from core-storage audit

Fixes verified defects found in a deep audit of the core storage path
(erasure coding, disk persistence, quorum, heal, replication resync):

- ecstore/disk: rewrite live xl.meta atomically (temp+rename) in
  delete_versions_internal and write_metadata instead of in-place
  truncate, which exposed torn metadata to concurrent readers and
  crashes on the DeleteObjects hot path
- ecstore/erasure: allow heal to reconstruct from exactly data_shards
  bitrot-verified sources; requiring data_shards+1 made objects
  permanently unhealable after losing parity_shards disks
- ecstore/set_disk: direct-memory inline GET applied the erasure
  distribution permutation twice (shuffled inputs re-indexed through
  distribution), concatenating wrong shards into the response body in
  degraded reads; collect from canonical disk-ordered inputs
- ecstore/set_disk: heal now preserves the committed inline layout
  instead of recomputing it with a hardcoded unversioned threshold,
  which split quorum identity of healed replicas and caused endless
  re-heal churn
- ecstore/replication: resync results channel switched from
  broadcast(1) to mpsc; a lagged broadcast receiver ended the stats
  collector and every subsequent failure went uncounted, letting
  failed resyncs be marked completed
- ecstore/replication: ignore an empty persisted resync checkpoint;
  resuming with one skipped every object and marked the resync
  completed without replicating anything
- ecstore/replication: fix inverted not-found error classification in
  replicate_object/replicate_delete logging paths
- ecstore/erasure: guard decode paths against zero block_size or
  data_shards from corrupt on-disk metadata (divide-by-zero panic)
- ecstore/disk: os::read_dir no longer consumes the entry limit on
  entries it does not return (is_empty_dir misjudgment); create_file
  opens with O_TRUNC to avoid stale trailing bytes
- filemeta: treat Some(nil) version id as a null version in
  matches_not_strict; disk-loaded headers never store None, so the
  mod_time quorum guard for unversioned overwrites never fired and an
  interrupted overwrite could displace the committed version in merge
- filemeta: fix msgpack skip lengths for fixext (missed the ext type
  byte) and ext16/32 (over-skipped) unknown fields
- filemeta: return FileCorrupt instead of usize underflow when
  xl.meta is truncated inside the CRC trailer
- filemeta: surface delete-marker insertion failure in delete_version
  instead of reporting success when the data dir is shared

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(replication): drop duplicate cfg(test) etag import from boundary module

The test module already imports content_matches_by_etag locally, so the
top-level cfg(test) import is unused under -D warnings and fails clippy.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

crates/ecstore/src/disk/local.rs

@@ -1623,12 +1623,11 @@ impl LocalDisk {
             return Ok(());
         }
 
-        // Update xl.meta
+        // Update xl.meta atomically: a concurrent reader or crash mid-write must
+        // never observe a truncated xl.meta for versions that were not deleted.
         let buf = fm.marshal_msg()?;
 
-        let volume_dir = self.get_bucket_path(volume)?;
-
-        self.write_all_private(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), buf.into(), true, &volume_dir)
+        self.write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
             .await?;
 
         Ok(())
@@ -2873,7 +2872,9 @@ impl DiskAPI for LocalDisk {
         if let Some(parent) = file_path.parent() {
             os::make_dir_all(parent, &volume_dir).await?;
         }
-        let f = super::fs::open_file(&file_path, O_CREATE | O_WRONLY)
+        // O_TRUNC: if a file already exists at this path, stale trailing bytes past
+        // the new content would otherwise survive and mismatch the metadata size.
+        let f = super::fs::open_file(&file_path, O_CREATE | O_WRONLY | O_TRUNC)
             .await
             .map_err(to_file_error)?;
         let reclaim_on_shutdown = should_reclaim_file_cache_after_write(_file_size);
@@ -3927,7 +3928,9 @@ impl DiskAPI for LocalDisk {
 
         let fm_data = meta.marshal_msg()?;
 
-        self.write_all(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), fm_data.into())
+        // Atomic temp+rename: this path also rewrites live xl.meta (delete markers,
+        // decommission), where an in-place truncate would expose torn metadata.
+        self.write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &fm_data, true)
             .await?;
 
         Ok(())

crates/ecstore/src/disk/os.rs

@@ -117,6 +117,10 @@ pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec
             volumes.push(name);
         } else if file_type.is_dir() {
             volumes.push(format!("{name}{SLASH_SEPARATOR}"));
+        } else {
+            // Entries we don't return (symlinks, sockets, fifos) must not consume
+            // the limit: is_empty_dir/list_dir(count=1) would misreport otherwise.
+            continue;
         }
         count -= 1;
         if count == 0 {