test(ecstore): trace the first unknown native publication cause

Record the first transition in test builds using existing lock and identity observations. Preserve the producer, rollback assertions, workflow, and test selection.

Validation: cargo fmt --all --check; cargo check -p rustfs-ecstore --tests; all 19 original quick-check gates, including make script-tests and both Python checks. Linux diagnostic execution is pending.

crates/ecstore/src/bucket/metadata_sys.rs

@@ -1649,15 +1649,39 @@ impl BucketMetadataSys {
     }
 
     pub(crate) async fn pending_bucket_creation(&self, bucket: &str) -> Option<PendingBucketCreationProof> {
-        self.pending_bucket_creations.read().await.get(bucket).cloned()
+        let observed = self.pending_bucket_creations.read().await.get(bucket).cloned();
+        #[cfg(test)]
+        if bucket.starts_with("real-guard-loss-") {
+            eprintln!(
+                "[creation-proof-diag] bucket={bucket} stage=pending-read present={} map={:p}",
+                observed.is_some(),
+                Arc::as_ptr(&self.pending_bucket_creations)
+            );
+        }
+        observed
     }
 
     pub(crate) async fn grant_pending_bucket_creation(&self, bucket: &str, proof: PendingBucketCreationProof) {
         self.pending_bucket_creations.write().await.insert(bucket.to_owned(), proof);
+        #[cfg(test)]
+        if bucket.starts_with("real-guard-loss-") {
+            eprintln!(
+                "[creation-proof-diag] bucket={bucket} stage=pending-inserted map={:p}",
+                Arc::as_ptr(&self.pending_bucket_creations)
+            );
+        }
     }
 
     pub(crate) async fn revoke_pending_bucket_creation(&self, bucket: &str) {
-        self.pending_bucket_creations.write().await.remove(bucket);
+        self.pending_bucket_creations.write().await.remove(bucket).inspect(|_| {
+            #[cfg(test)]
+            if bucket.starts_with("real-guard-loss-") {
+                eprintln!(
+                    "[creation-proof-diag] bucket={bucket} stage=pending-removed-by-revoke map={:p}",
+                    Arc::as_ptr(&self.pending_bucket_creations)
+                );
+            }
+        });
     }
 
     pub(crate) async fn consume_pending_bucket_creation(

crates/ecstore/src/disk/local.rs

@@ -10830,9 +10830,27 @@ impl DiskAPI for LocalDisk {
     #[tracing::instrument(level = "trace", skip_all)]
     async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
         let Some(history) = self.publication_history.as_ref() else {
+            #[cfg(test)]
+            if volume.starts_with("real-guard-loss-") {
+                eprintln!("[creation-proof-diag] bucket={volume} stage=local-history-unavailable");
+            }
             return Ok(None);
         };
-        let Some(disk_id) = self.get_disk_id().await? else {
+        let observed = self.get_disk_id().await;
+        #[cfg(test)]
+        if volume.starts_with("real-guard-loss-")
+            && let Err(error) = &observed
+        {
+            eprintln!(
+                "[creation-proof-diag] bucket={volume} stage=local-disk-id-error discriminant={:?}",
+                std::mem::discriminant(error)
+            );
+        }
+        let Some(disk_id) = observed? else {
+            #[cfg(test)]
+            if volume.starts_with("real-guard-loss-") {
+                eprintln!("[creation-proof-diag] bucket={volume} stage=local-disk-id-unavailable");
+            }
             return Ok(None);
         };
         Ok(history.witness(volume, disk_id, Arc::clone(&self.io_backend)))

crates/ecstore/src/disk/os.rs

@@ -1407,6 +1407,38 @@ struct NativePublicationHistory {
 struct NativeRootHistory {
     unknown: bool,
     started: HashSet<String>,
+    #[cfg(test)]
+    first_unknown: Option<NativeFirstUnknown>,
+}
+
+#[cfg(test)]
+#[derive(Debug)]
+struct NativeFirstUnknown {
+    reason: &'static str,
+    aliases: [Option<PathBuf>; 2],
+    old_identity: Option<NativeRootIdentity>,
+    observed_identities: [Option<NativeRootIdentity>; 2],
+}
+
+#[cfg(test)]
+impl NativeRootHistory {
+    fn record_first_unknown(
+        &mut self,
+        was_unknown: bool,
+        reason: &'static str,
+        aliases: [Option<&Path>; 2],
+        old_identity: Option<NativeRootIdentity>,
+        observed_identities: [Option<NativeRootIdentity>; 2],
+    ) {
+        if !was_unknown && self.first_unknown.is_none() {
+            self.first_unknown = Some(NativeFirstUnknown {
+                reason,
+                aliases: aliases.map(|alias| alias.map(Path::to_path_buf)),
+                old_identity,
+                observed_identities,
+            });
+        }
+    }
 }
 
 static NATIVE_PUBLICATION_HISTORY: LazyLock<Mutex<NativePublicationHistory>> =
@@ -1417,6 +1449,8 @@ pub(crate) struct NativePublicationRegistration {
     identity: NativeRootIdentity,
     root: PathBuf,
     io_root: PathBuf,
+    #[cfg(test)]
+    diagnostic_previous_unknown: Option<bool>,
 }
 
 /// Opaque, process-local observation. Unsupported disks cannot construct it.
@@ -1457,16 +1491,44 @@ pub(crate) fn register_native_publication_root(root: &Path, io_root: &Path) -> O
         return None;
     }
     let mut history = NATIVE_PUBLICATION_HISTORY.lock();
+    #[cfg(test)]
+    let diagnostic_previous_unknown = history.roots.get(&identity).map(|root| root.unknown);
     history.roots.entry(identity).or_default();
     for alias in [root, io_root] {
         let alias: PathBuf = alias.components().collect();
+        #[cfg(test)]
+        let diagnostic_alias = alias.clone();
         let previous = history.aliases.entry(alias).or_default();
         let collisions: Vec<_> = previous.iter().copied().filter(|old| *old != identity).collect();
         previous.insert(identity);
         if !collisions.is_empty() {
+            #[cfg(test)]
+            let diagnostic_was_unknown = history.roots.get(&identity).is_some_and(|root| root.unknown);
             history.roots.get_mut(&identity)?.unknown = true;
+            #[cfg(test)]
+            if let Some(root) = history.roots.get_mut(&identity) {
+                root.record_first_unknown(
+                    diagnostic_was_unknown,
+                    "registration-alias-collision",
+                    [Some(&diagnostic_alias), None],
+                    collisions.first().copied(),
+                    [Some(identity), Some(identity)],
+                );
+            }
             for old in collisions {
+                #[cfg(test)]
+                let diagnostic_was_unknown = history.roots.get(&old).is_some_and(|root| root.unknown);
                 history.roots.get_mut(&old)?.unknown = true;
+                #[cfg(test)]
+                if let Some(root) = history.roots.get_mut(&old) {
+                    root.record_first_unknown(
+                        diagnostic_was_unknown,
+                        "registration-alias-collision",
+                        [Some(&diagnostic_alias), None],
+                        Some(old),
+                        [Some(identity), Some(identity)],
+                    );
+                }
             }
         }
     }
@@ -1474,6 +1536,8 @@ pub(crate) fn register_native_publication_root(root: &Path, io_root: &Path) -> O
         identity,
         root: root.to_path_buf(),
         io_root: io_root.to_path_buf(),
+        #[cfg(test)]
+        diagnostic_previous_unknown,
     })
 }
 
@@ -1509,20 +1573,60 @@ pub(crate) fn record_native_publication(path: &Path) {
         // with the stat, rather than using a stale captured identity list.
         let identities = history.aliases.get(&alias).cloned().unwrap_or_default();
         if physical.is_none() || identities.iter().any(|identity| Some(*identity) != physical) {
+            #[cfg(test)]
+            let diagnostic_old_identity = identities.iter().find(|identity| Some(**identity) != physical).copied();
             for identity in identities {
+                #[cfg(test)]
+                let diagnostic_was_unknown = history.roots.get(&identity).is_some_and(|root| root.unknown);
                 history.roots.entry(identity).or_default().unknown = true;
+                #[cfg(test)]
+                if let Some(root) = history.roots.get_mut(&identity) {
+                    root.record_first_unknown(
+                        diagnostic_was_unknown,
+                        "publication-alias-identity-mismatch",
+                        [Some(&alias), None],
+                        Some(identity),
+                        [physical, None],
+                    );
+                }
             }
             if let Some(identity) = physical {
+                #[cfg(test)]
+                let diagnostic_was_unknown = history.roots.get(&identity).is_some_and(|root| root.unknown);
                 history.roots.entry(identity).or_default().unknown = true;
+                #[cfg(test)]
+                if let Some(root) = history.roots.get_mut(&identity) {
+                    root.record_first_unknown(
+                        diagnostic_was_unknown,
+                        "publication-alias-identity-mismatch",
+                        [Some(&alias), None],
+                        diagnostic_old_identity,
+                        [physical, None],
+                    );
+                }
             }
         } else if let Some(identity) = physical {
             let root = history.roots.entry(identity).or_default();
+            #[cfg(test)]
+            let diagnostic_was_unknown = root.unknown;
+            #[cfg(test)]
+            let diagnostic_invalid_bucket = bucket.is_none();
             match bucket {
                 Some(bucket) => {
                     root.started.insert(bucket);
                 }
                 None => root.unknown = true,
             }
+            #[cfg(test)]
+            if diagnostic_invalid_bucket {
+                root.record_first_unknown(
+                    diagnostic_was_unknown,
+                    "publication-invalid-relative-bucket",
+                    [Some(&alias), None],
+                    Some(identity),
+                    [physical, None],
+                );
+            }
         }
     }
 }
@@ -1530,7 +1634,17 @@ pub(crate) fn record_native_publication(path: &Path) {
 impl NativePublicationRegistration {
     pub(crate) fn invalidate(&self) {
         if let Some(root) = NATIVE_PUBLICATION_HISTORY.lock().roots.get_mut(&self.identity) {
+            #[cfg(test)]
+            let diagnostic_was_unknown = root.unknown;
             root.unknown = true;
+            #[cfg(test)]
+            root.record_first_unknown(
+                diagnostic_was_unknown,
+                "registration-invalidate",
+                [Some(&self.root), Some(&self.io_root)],
+                Some(self.identity),
+                [None, None],
+            );
         }
     }
 
@@ -1544,6 +1658,10 @@ impl NativePublicationRegistration {
             || !matches!(Path::new(bucket).components().next(), Some(Component::Normal(_)))
             || Path::new(bucket).components().count() != 1
         {
+            #[cfg(test)]
+            if bucket.starts_with("real-guard-loss-") {
+                eprintln!("[creation-proof-diag] bucket={bucket} stage=native-input-rejected");
+            }
             return None;
         }
         let witness = NativeBucketCreationWitness {
@@ -1557,6 +1675,25 @@ impl NativePublicationRegistration {
 }
 
 impl NativeBucketCreationWitness {
+    #[cfg(test)]
+    fn log_first_unknown(&self, root: &NativeRootHistory) {
+        if !self.bucket.starts_with("real-guard-loss-") {
+            return;
+        }
+        let first_unknown = root
+            .first_unknown
+            .as_ref()
+            .map(|cause| (cause.reason, &cause.aliases, cause.old_identity, cause.observed_identities));
+        eprintln!(
+            "[creation-proof-diag] bucket={} stage=native-history-first-unknown key={:?} first_unknown={first_unknown:?} configured_root={:?} configured_io_root={:?} registration_previous_unknown={:?}",
+            self.bucket,
+            self.registration.identity,
+            self.registration.root,
+            self.registration.io_root,
+            self.registration.diagnostic_previous_unknown,
+        );
+    }
+
     pub(crate) fn matches(&self, other: &Self) -> bool {
         self.registration.identity == other.registration.identity
             && Arc::ptr_eq(&self.backend, &other.backend)
@@ -1567,22 +1704,83 @@ impl NativeBucketCreationWitness {
     fn physical_root_is_current(&self) -> bool {
         let root = native_root_identity(&self.registration.root);
         let io_root = native_root_identity(&self.registration.io_root);
+        #[cfg(test)]
+        if self.bucket.starts_with("real-guard-loss-") {
+            eprintln!(
+                "[creation-proof-diag] bucket={} stage=native-physical-root expected={:?} observed_root={root:?} observed_io_root={io_root:?} backend={:p}",
+                self.bucket,
+                self.registration.identity,
+                Arc::as_ptr(&self.backend)
+            );
+        }
         if root == Some(self.registration.identity) && io_root == Some(self.registration.identity) {
             return true;
         }
         let mut history = NATIVE_PUBLICATION_HISTORY.lock();
+        #[cfg(test)]
+        let diagnostic_was_unknown = history
+            .roots
+            .get(&self.registration.identity)
+            .is_some_and(|root| root.unknown);
         history.roots.entry(self.registration.identity).or_default().unknown = true;
+        #[cfg(test)]
+        if let Some(state) = history.roots.get_mut(&self.registration.identity) {
+            state.record_first_unknown(
+                diagnostic_was_unknown,
+                "witness-physical-root-drift",
+                [Some(&self.registration.root), Some(&self.registration.io_root)],
+                Some(self.registration.identity),
+                [root, io_root],
+            );
+        }
         for identity in [root, io_root].into_iter().flatten() {
+            #[cfg(test)]
+            let diagnostic_was_unknown = history.roots.get(&identity).is_some_and(|root| root.unknown);
             history.roots.entry(identity).or_default().unknown = true;
+            #[cfg(test)]
+            if let Some(state) = history.roots.get_mut(&identity) {
+                state.record_first_unknown(
+                    diagnostic_was_unknown,
+                    "witness-physical-root-drift",
+                    [Some(&self.registration.root), Some(&self.registration.io_root)],
+                    Some(self.registration.identity),
+                    [root, io_root],
+                );
+            }
+        }
+        #[cfg(test)]
+        if let Some(state) = history.roots.get(&self.registration.identity) {
+            self.log_first_unknown(state);
         }
         false
     }
 
     fn valid_in(&self, history: &NativePublicationHistory) -> bool {
-        history
-            .roots
-            .get(&self.registration.identity)
-            .is_some_and(|root| !root.unknown && !root.started.contains(&self.bucket))
+        let Some(root) = history.roots.get(&self.registration.identity) else {
+            #[cfg(test)]
+            if self.bucket.starts_with("real-guard-loss-") {
+                eprintln!("[creation-proof-diag] bucket={} stage=native-history-missing", self.bucket);
+            }
+            return false;
+        };
+        let known = !root.unknown;
+        #[cfg(test)]
+        self.log_first_unknown(root);
+        #[cfg(test)]
+        if self.bucket.starts_with("real-guard-loss-") {
+            eprintln!("[creation-proof-diag] bucket={} stage=native-history-known known={known}", self.bucket);
+        }
+        known && {
+            let unstarted = !root.started.contains(&self.bucket);
+            #[cfg(test)]
+            if self.bucket.starts_with("real-guard-loss-") {
+                eprintln!(
+                    "[creation-proof-diag] bucket={} stage=native-history-unstarted unstarted={unstarted}",
+                    self.bucket
+                );
+            }
+            unstarted
+        }
     }
 }
 

crates/ecstore/src/store/bucket.rs

@@ -468,14 +468,51 @@ impl ECStore {
         for (_, _, set) in self.bucket_sets() {
             let disks = set.disks.read().await.clone();
             if disks.is_empty() {
+                #[cfg(test)]
+                if bucket.starts_with("real-guard-loss-") {
+                    eprintln!("[creation-proof-diag] bucket={bucket} stage=native-empty-set");
+                }
                 return None;
             }
             for disk in disks {
+                #[cfg(test)]
+                if bucket.starts_with("real-guard-loss-") && disk.is_none() {
+                    eprintln!("[creation-proof-diag] bucket={bucket} stage=native-missing-disk");
+                }
                 let disk = disk?;
-                if !matches!(disk.stat_volume(bucket).await, Err(DiskError::VolumeNotFound)) {
+                let volume_absent = {
+                    let observed = disk.stat_volume(bucket).await;
+                    #[cfg(test)]
+                    if bucket.starts_with("real-guard-loss-") {
+                        match &observed {
+                            Ok(_) => eprintln!("[creation-proof-diag] bucket={bucket} stage=native-volume-present"),
+                            Err(error) if !matches!(error, DiskError::VolumeNotFound) => eprintln!(
+                                "[creation-proof-diag] bucket={bucket} stage=native-stat-error faulty_disk={} discriminant={:?}",
+                                matches!(error, DiskError::FaultyDisk),
+                                std::mem::discriminant(error)
+                            ),
+                            Err(_) => {}
+                        }
+                    }
+                    matches!(observed, Err(DiskError::VolumeNotFound))
+                };
+                if !volume_absent {
                     return None;
                 }
-                witnesses.push(disk.bucket_creation_witness(bucket).await.ok()??);
+                let observed = disk.bucket_creation_witness(bucket).await;
+                #[cfg(test)]
+                if bucket.starts_with("real-guard-loss-") {
+                    match &observed {
+                        Err(error) => eprintln!(
+                            "[creation-proof-diag] bucket={bucket} stage=native-witness-error faulty_disk={} discriminant={:?}",
+                            matches!(error, DiskError::FaultyDisk),
+                            std::mem::discriminant(error)
+                        ),
+                        Ok(None) => eprintln!("[creation-proof-diag] bucket={bucket} stage=native-witness-none"),
+                        Ok(Some(_)) => {}
+                    }
+                }
+                witnesses.push(observed.ok()??);
             }
         }
         crate::disk::os::validate_native_creation_witnesses(&witnesses).then_some(witnesses)
@@ -492,29 +529,95 @@ impl ECStore {
     where
         F: Fn() -> bool + Send,
     {
-        let after = self.native_creation_witnesses(bucket).await?;
+        let after = self.native_creation_witnesses(bucket).await;
+        #[cfg(test)]
+        if bucket.starts_with("real-guard-loss-") {
+            eprintln!(
+                "[creation-proof-diag] bucket={bucket} stage=after-witness-result available={}",
+                after.is_some()
+            );
+        }
+        let after = after?;
         if before.len() != after.len() || !before.iter().zip(&after).all(|(old, current)| old.matches(current)) {
+            #[cfg(test)]
+            if bucket.starts_with("real-guard-loss-") {
+                eprintln!("[creation-proof-diag] bucket={bucket} stage=witness-pair-rejected");
+            }
             return None;
         }
-        let (persisted, present) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket)
-            .await
-            .ok()?;
+        let loaded = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await;
+        #[cfg(test)]
+        if bucket.starts_with("real-guard-loss-")
+            && let Err(error) = &loaded
+        {
+            eprintln!(
+                "[creation-proof-diag] bucket={bucket} stage=metadata-load-error discriminant={:?}",
+                std::mem::discriminant(error)
+            );
+        }
+        let (persisted, present) = loaded.ok()?;
         let mut expected = metadata.clone();
         expected.bucket_creation_committed = false;
         expected.bucket_creation_commit_record_present = true;
         expected.bucket_incarnation_sidecar = true;
         expected.default_timestamps();
-        let intent = persisted.marshal_msg().ok()?;
+        let encoded = persisted.marshal_msg();
+        #[cfg(test)]
+        if bucket.starts_with("real-guard-loss-")
+            && let Err(error) = &encoded
+        {
+            eprintln!(
+                "[creation-proof-diag] bucket={bucket} stage=persisted-encode-error discriminant={:?}",
+                std::mem::discriminant(error)
+            );
+        }
+        let intent = encoded.ok()?;
         if !present
             || persisted.bucket_creation_committed
             || !persisted.bucket_incarnation_sidecar
-            || intent != expected.marshal_msg().ok()?
+            || intent != {
+                let encoded = expected.marshal_msg();
+                #[cfg(test)]
+                if bucket.starts_with("real-guard-loss-")
+                    && let Err(error) = &encoded
+                {
+                    eprintln!(
+                        "[creation-proof-diag] bucket={bucket} stage=expected-encode-error discriminant={:?}",
+                        std::mem::discriminant(error)
+                    );
+                }
+                encoded.ok()?
+            }
         {
+            #[cfg(test)]
+            if bucket.starts_with("real-guard-loss-") {
+                eprintln!(
+                    "[creation-proof-diag] bucket={bucket} stage=metadata-shape-or-intent-rejected present={present} committed={} sidecar={}",
+                    persisted.bucket_creation_committed, persisted.bucket_incarnation_sidecar
+                );
+            }
             return None;
         }
-        let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx).ok()?;
+        let system = metadata_sys::require_bucket_metadata_sys_in(&self.ctx);
+        #[cfg(test)]
+        if bucket.starts_with("real-guard-loss-")
+            && let Err(error) = &system
+        {
+            eprintln!(
+                "[creation-proof-diag] bucket={bucket} stage=metadata-system-error discriminant={:?}",
+                std::mem::discriminant(error)
+            );
+        }
+        let sys = system.ok()?;
         let sys = sys.read().await.clone();
-        if !fences_valid() || !crate::disk::os::validate_native_creation_witnesses(&after) {
+        let current_fences_valid = fences_valid();
+        if !current_fences_valid || !crate::disk::os::validate_native_creation_witnesses(&after) {
+            #[cfg(test)]
+            if bucket.starts_with("real-guard-loss-") {
+                eprintln!(
+                    "[creation-proof-diag] bucket={bucket} stage=final-fence-or-history-rejected fences_valid={current_fences_valid}"
+                );
+            }
             return None;
         }
         sys.grant_pending_bucket_creation(
@@ -1336,11 +1439,24 @@ impl ECStore {
                 } else {
                     false
                 };
+                #[cfg(test)]
+                if bucket.starts_with("real-guard-loss-") {
+                    eprintln!(
+                        "[creation-proof-diag] bucket={bucket} stage=prior-intent-observation absent={prior_intent_absent}"
+                    );
+                }
                 let before = if prior_intent_absent {
                     self.native_creation_witnesses(bucket).await
                 } else {
                     None
                 };
+                #[cfg(test)]
+                if bucket.starts_with("real-guard-loss-") {
+                    eprintln!(
+                        "[creation-proof-diag] bucket={bucket} stage=before-witness-result available={}",
+                        before.is_some()
+                    );
+                }
                 await_bucket_lifecycle_operation(
                     bucket_lifecycle_guard.as_ref(),
                     ns_guard.as_ref(),