fix(ecstore): never cache fabricated bucket metadata as authoritative (#5307)

BucketMetadataSys::get_config lazily fabricated a default BucketMetadata
(object-lock off) for any bucket whose .metadata.bin was ConfigNotFound and
cached it in the map that the map-only, fail-closed metadata_sys::get()
serves. The object-lock batch-delete gate (object_lock_delete_check_required,
backlog#929 / #4297) treats that map as authoritative, so a metadata miss
became a cached "no lock" answer: a versioning peek could poison the cache
and let delete_objects skip the per-object retention/legal-hold stat. The
same fabrication raced make_bucket (lost update overwriting freshly
persisted lock-enabled metadata) and let the 15-minute refresh loop replace
good cached metadata on a transient quorum dip.

Production changes:
- get_config caches only metadata actually read from disk; misses are
  recorded in a bounded negative cache (30s TTL, 10k entries, invalidated by
  set()) so repeated lookups for metadata-less names cost no extra
  namespace-lock + erasure-set fanout (reachable pre-auth via CORS
  preflight and per-key in DeleteObjects).
- concurrent_load never lets a fabricated default REPLACE an existing map
  entry; startup insert-if-vacant behavior for legacy buckets is preserved.
- delete_objects and new_ns_lock resolve dist-erasure, versioning, and the
  object-lock gate from the set's own instance context (backlog#1052)
  instead of the ambient facade, so a second in-process instance (or, in
  tests, another test's transient DistErasure window) cannot reroute
  locking onto an empty dist locker list or answer with the wrong
  instance's bucket state.

Test-isolation changes (the bug that surfaced all of the above: the
delete_objects lock-gating test failed deterministically when sharing a
process with the lifecycle env tests):
- The MinIO-migration test builds on an isolated InstanceContext instead of
  registering soon-deleted disks in the shared bootstrap registry.
- The cached lifecycle env re-registers its disks on every use, surviving
  other serial tests' reset_local_disk_test_state.
- Hermetic SetDisks helpers gain isolated-context variants pinned to plain
  erasure; tier-free non-serial test modules use them, guard-based
  SetupTypeGuard tests stay on the bootstrap context.
- Three deterministic pin tests (nextest-safe) cover the caching contract,
  the delete gate resolution source, and the ns-lock resolution source.

Verification:
- cargo test -p rustfs-ecstore --lib -- --exact <4-test combo from the
  report> (previously failing, now green)
- cargo test -p rustfs-ecstore --lib: 3169 passed / 0 failed across
  repeated runs; cargo fmt --check and cargo clippy --lib --tests clean
- Adversarial validation (high-risk tier, all seven roles) run per
  AGENTS.md; all findings fixed or rebutted with evidence

crates/ecstore/src/set_disk/mod.rs

@@ -4596,12 +4596,19 @@ mod tests {
     }
 
     async fn make_test_set_disks(lockers: Vec<Arc<dyn LockClient>>) -> Arc<SetDisks> {
+        make_test_set_disks_with_ctx(lockers, bootstrap_ctx()).await
+    }
+
+    async fn make_test_set_disks_with_ctx(
+        lockers: Vec<Arc<dyn LockClient>>,
+        instance_ctx: Arc<InstanceContext>,
+    ) -> Arc<SetDisks> {
         let endpoints = vec![
             Endpoint::try_from("http://127.0.0.1:9000/data").expect("first endpoint should parse"),
             Endpoint::try_from("http://127.0.0.1:9001/data").expect("second endpoint should parse"),
         ];
 
-        SetDisks::new(
+        SetDisks::new_with_instance_ctx(
             "test-owner".to_string(),
             Arc::new(RwLock::new(vec![None, None])),
             2,
@@ -4611,10 +4618,53 @@ mod tests {
             endpoints,
             FormatV3::new(1, 2),
             lockers,
+            instance_ctx,
         )
         .await
     }
 
+    /// Pins the dist-erasure resolution SOURCE for `new_ns_lock` (adversarial
+    /// review): the lock strategy must come from the set's own instance
+    /// context, never the ambient facade — otherwise another in-process
+    /// instance (or a concurrent test's ambient DistErasure window) reroutes
+    /// this set's locking onto the wrong strategy.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn new_ns_lock_resolves_dist_from_set_instance_context() {
+        let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
+        let locker: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager));
+
+        let dist_ctx = Arc::new(InstanceContext::new());
+        dist_ctx.update_erasure_type(SetupType::DistErasure).await;
+        let dist_set = make_test_set_disks_with_ctx(vec![locker.clone()], dist_ctx).await;
+        let dist_guard = dist_set
+            .new_ns_lock("bucket", "object")
+            .await
+            .expect("namespace lock should be created")
+            .get_read_lock(Duration::from_millis(500))
+            .await
+            .expect("dist read lock should succeed with one healthy locker");
+        assert!(
+            matches!(dist_guard, NamespaceLockGuard::Standard(_)),
+            "a DistErasure instance context must select the distributed lock strategy"
+        );
+        drop(dist_guard);
+
+        let local_ctx = Arc::new(InstanceContext::new());
+        local_ctx.update_erasure_type(SetupType::Erasure).await;
+        let local_set = make_test_set_disks_with_ctx(vec![locker], local_ctx).await;
+        let local_guard = local_set
+            .new_ns_lock("bucket", "object")
+            .await
+            .expect("namespace lock should be created")
+            .get_read_lock(Duration::from_millis(500))
+            .await
+            .expect("local read lock should succeed");
+        assert!(
+            matches!(local_guard, NamespaceLockGuard::Fast(_)),
+            "a plain-erasure instance context must select the local lock strategy"
+        );
+    }
+
     struct SetupTypeGuard {
         previous: SetupType,
     }

crates/ecstore/src/set_disk/ops/locking.rs

@@ -30,7 +30,12 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
 
     #[tracing::instrument(skip(self))]
     async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
-        let set_lock = if runtime_sources::setup_is_dist_erasure().await {
+        // Resolved from this set's own instance context (backlog#1052), not the
+        // ambient facade: the facade tracks whichever context is currently
+        // published process-wide, so a second instance (or, in tests, another
+        // test's transient DistErasure window) would push this set's namespace
+        // locking onto its own — possibly empty — dist locker list.
+        let set_lock = if self.ctx.is_dist_erasure().await {
             // Calculate quorum based on lockers count (majority)
             let lockers_count = self.lockers.len();
             let write_quorum = if lockers_count > 1 { (lockers_count / 2) + 1 } else { 1 };

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -1898,8 +1898,13 @@ mod tests {
     use crate::disk::DiskAPI as _;
     use crate::disk::{endpoint::Endpoint, format::FormatV3};
     use crate::layout::endpoints::SetupType;
+    // No-locker helpers resolve to the isolated-context variants (see
+    // `hermetic_set_disks_isolated`); the guard-based tests build through
+    // `hermetic_set_disks_with_lockers`, which stays on the bootstrap context
+    // so their own ambient SetupTypeGuard flip remains visible to them.
     use crate::set_disk::ops::object::hermetic_set_disks_support::{
-        hermetic_set_disks, hermetic_set_disks_for_pool_with_default_parity, hermetic_set_disks_with_lockers,
+        hermetic_set_disks_for_pool_with_default_parity_isolated as hermetic_set_disks_for_pool_with_default_parity,
+        hermetic_set_disks_isolated as hermetic_set_disks, hermetic_set_disks_with_lockers,
     };
     use crate::storage_api_contracts::namespace::NamespaceLocking as _;
     use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};

crates/ecstore/src/set_disk/ops/object.rs

@@ -2671,7 +2671,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
         let mut locked_objects = HashSet::new();
 
-        let dist_erasure = runtime_sources::setup_is_dist_erasure().await;
+        // Instance-scoped, not the ambient facade (backlog#1052) — see
+        // new_ns_lock. The same applies to the versioning and bucket-metadata
+        // lookups below: resolving them through the first published
+        // instance's context would let a second in-process store delete with
+        // the wrong versioning semantics or skip the object-lock gate.
+        let dist_erasure = self.ctx.is_dist_erasure().await;
         let mut dist_batch_lock_ids = vec![Vec::new(); self.lockers.len()];
 
         if opts.no_lock {
@@ -2716,7 +2721,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
-        let ver_cfg = BucketVersioningSys::get(bucket).await.unwrap_or_default();
+        let ver_cfg = BucketVersioningSys::get_in(&self.ctx, bucket).await.unwrap_or_default();
 
         // backlog#929 (HP-8): the per-object stat below exists solely to feed
         // check_object_lock_delete (#4297). Resolve the bucket lock
@@ -2724,7 +2729,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         // for buckets without Object Lock; unknown metadata fails closed and
         // keeps the stat, so the #4297 protection is preserved verbatim for
         // every object-lock-enabled bucket.
-        let object_lock_checks_required = object_lock_delete_check_required(metadata_sys::get(bucket).await.ok().as_deref());
+        let object_lock_checks_required =
+            object_lock_delete_check_required(metadata_sys::get_in(&self.ctx, bucket).await.ok().as_deref());
 
         let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
 
@@ -4098,11 +4104,57 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
         hermetic_set_disks_with_lockers(disk_count, pool_index, default_parity_count, Vec::new()).await
     }
 
+    /// Like [`hermetic_set_disks`], but binds the set to an isolated instance
+    /// context pinned to plain erasure. `#[serial]` tests elsewhere flip the
+    /// shared bootstrap context to DistErasure (`SetupTypeGuard`) while
+    /// non-serial hermetic tests run; on the shared context such a window
+    /// reroutes locking onto the empty dist locker list ("No lock clients
+    /// available") and the batch-delete gate onto the dist path. Only suitable
+    /// for tests that never touch context-resolved services registered on the
+    /// ambient context (tier config manager, expiry state, ...), because the
+    /// isolated context starts every one of those cells fresh.
+    pub(in crate::set_disk::ops) async fn hermetic_set_disks_isolated(
+        disk_count: usize,
+    ) -> (Vec<TempDir>, Vec<DiskStore>, Arc<SetDisks>) {
+        hermetic_set_disks_for_pool_with_default_parity_isolated(disk_count, 0, disk_count / 2).await
+    }
+
+    /// Pool-parameterized variant of [`hermetic_set_disks_isolated`] with the
+    /// same isolation contract.
+    pub(in crate::set_disk::ops) async fn hermetic_set_disks_for_pool_with_default_parity_isolated(
+        disk_count: usize,
+        pool_index: usize,
+        default_parity_count: usize,
+    ) -> (Vec<TempDir>, Vec<DiskStore>, Arc<SetDisks>) {
+        let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        isolated_ctx
+            .update_erasure_type(crate::layout::endpoints::SetupType::Erasure)
+            .await;
+        hermetic_set_disks_with_lockers_and_ctx(disk_count, pool_index, default_parity_count, Vec::new(), isolated_ctx).await
+    }
+
     pub(in crate::set_disk::ops) async fn hermetic_set_disks_with_lockers(
         disk_count: usize,
         pool_index: usize,
         default_parity_count: usize,
         lockers: Vec<Arc<dyn LockClient>>,
+    ) -> (Vec<TempDir>, Vec<DiskStore>, Arc<SetDisks>) {
+        hermetic_set_disks_with_lockers_and_ctx(
+            disk_count,
+            pool_index,
+            default_parity_count,
+            lockers,
+            crate::runtime::instance::bootstrap_ctx(),
+        )
+        .await
+    }
+
+    pub(in crate::set_disk::ops) async fn hermetic_set_disks_with_lockers_and_ctx(
+        disk_count: usize,
+        pool_index: usize,
+        default_parity_count: usize,
+        lockers: Vec<Arc<dyn LockClient>>,
+        instance_ctx: Arc<crate::runtime::instance::InstanceContext>,
     ) -> (Vec<TempDir>, Vec<DiskStore>, Arc<SetDisks>) {
         let format = FormatV3::new(1, disk_count);
 
@@ -4119,7 +4171,7 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
             disks.push(Some(disk));
         }
 
-        let set_disks = SetDisks::new(
+        let set_disks = SetDisks::new_with_instance_ctx(
             "hermetic-ops-test-owner".to_string(),
             Arc::new(RwLock::new(disks)),
             disk_count,
@@ -4129,6 +4181,7 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
             endpoints,
             format,
             lockers,
+            instance_ctx,
         )
         .await;
 
@@ -4237,7 +4290,7 @@ mod get_object_downstream_close_accounting_tests {
 
 #[cfg(test)]
 mod metadata_mutation_generation_tests {
-    use super::hermetic_set_disks_support::hermetic_set_disks;
+    use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
     use super::*;
     use crate::disk::DiskAPI as _;
     use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
@@ -6587,7 +6640,7 @@ mod transition_source_identity_matrix_tests {
 
 #[cfg(test)]
 mod heterogeneous_pool_put_tests {
-    use super::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity;
+    use super::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated as hermetic_set_disks_for_pool_with_default_parity;
     use super::*;
     use crate::config::storageclass::lookup_config_for_pools_without_env;
     use crate::disk::{DiskAPI as _, ReadOptions};
@@ -6647,7 +6700,7 @@ mod put_object_tmp_cleanup_tests {
     //! response path), while a failed PUT must still clean its tmp shards
     //! inline before returning.
 
-    use super::hermetic_set_disks_support::hermetic_set_disks;
+    use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
     use super::*;
     use crate::disk::DiskAPI as _;
     use std::time::Duration;
@@ -6749,7 +6802,7 @@ mod put_object_tags_early_stop_regression_tests {
     //! with early-stop enabled, the tag must land on EVERY online disk's xl.meta,
     //! not a read-quorum subset.
 
-    use super::hermetic_set_disks_support::hermetic_set_disks;
+    use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
     use super::*;
     use crate::disk::{DiskAPI as _, ReadOptions};
 
@@ -6811,7 +6864,8 @@ mod delete_objects_lock_gating_tests {
     //! locked-stat path and prove the #4297 delete protection is intact end to
     //! end, while per-key result mapping of mixed batches stays stable.
 
-    use super::hermetic_set_disks_support::hermetic_set_disks;
+    use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
+    use super::hermetic_set_disks_support::hermetic_set_disks_with_lockers_and_ctx;
     use super::*;
     use crate::disk::DiskAPI as _;
 
@@ -6943,6 +6997,63 @@ mod delete_objects_lock_gating_tests {
             .expect_err("plain object must be deleted");
     }
 
+    /// Pins the dist-erasure resolution SOURCE for the batch-delete lock gate
+    /// (adversarial review): the decision must come from the set's own
+    /// instance context. With a DistErasure context and no dist lockers the
+    /// batch must fail closed on lock acquisition; ambient resolution
+    /// (non-dist in this process) would take local locks and let the delete
+    /// through.
+    #[tokio::test]
+    async fn delete_objects_dist_gate_uses_set_instance_context() {
+        let dist_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        dist_ctx
+            .update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
+            .await;
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers_and_ctx(4, 0, 2, Vec::new(), dist_ctx).await;
+        let bucket = "dist-gate-ctx-bucket";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        // The put must bypass locking: with a DistErasure context and no
+        // lockers, every namespace lock acquisition fails closed by design.
+        let mut reader = PutObjReader::from_vec(vec![5u8; 256]);
+        set_disks
+            .put_object(
+                bucket,
+                "obj",
+                &mut reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("object should be written without locks");
+
+        let objects = vec![ObjectToDelete {
+            object_name: "obj".to_string(),
+            ..Default::default()
+        }];
+        let (_deleted, errs) = set_disks.delete_objects(bucket, objects, ObjectOptions::default()).await;
+
+        assert!(
+            errs[0].is_some(),
+            "the dist gate resolved from the set's own context must fail closed on an empty locker list"
+        );
+        set_disks
+            .get_object_info(
+                bucket,
+                "obj",
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("object must survive the failed batch delete");
+    }
+
     #[tokio::test]
     async fn delete_objects_honors_no_lock_when_outer_write_lock_is_held() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;

crates/ecstore/src/set_disk/read.rs

@@ -2611,7 +2611,14 @@ mod metadata_cache_tests {
     #[tokio::test]
     #[serial(metadata_cache_publish_barrier)]
     async fn metadata_cache_production_fanout_cannot_publish_after_invalidation() {
-        let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks(4, 2).await;
+        // Isolated context: an ambient DistErasure window (another test's
+        // SetupTypeGuard) would bypass metadata-cache publication entirely
+        // and time out the barrier below.
+        let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        isolated_ctx
+            .update_erasure_type(crate::layout::endpoints::SetupType::Erasure)
+            .await;
+        let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await;
         let bucket = "metadata-cache-production-fence";
         let object = "object";
         let disks = set.disks.read().await.clone();