feat(ecstore): run bucket operations on the store's own instance context (#4642)

backlog#1052 S7 — the final piece: full bucket-namespace isolation
between embedded servers in one process.

Server B's requests resolved B's own ECStore (per-server dispatch landed
earlier), but the store's bucket operations still went through ambient
process facades, so both servers effectively operated on the FIRST
server's disks and metadata:

- LocalPeerS3Client::local_disks_for_pools() called all_local_disk()
  (the ambient disk registry = the first published store's context), so
  list/make/delete/heal bucket scanned and wrote the wrong volumes.
- BucketMetadata::save() persisted through the ambient object handle, so
  a second server's bucket metadata landed in the first server's
  .rustfs.sys; set/remove/get/created_at all used the ambient metadata
  system.

Now the whole chain is bound to the owning store's InstanceContext:

- S3PeerSys/LocalPeerS3Client gain *_with_instance_ctx constructors and
  operate on that context's registered disks; ECStore::new (and the test
  store builder) pass the store's context. The legacy constructors keep
  the bootstrap default.
- BucketMetadata::save_with_store persists through an explicit store;
  BucketMetadataSys::persist_and_set uses the system's own api handle.
- metadata_sys gains instance-scoped variants (get_in / created_at_in /
  set_bucket_metadata_in / remove_bucket_metadata_in) that resolve the
  context's metadata system and fall back to the ambient default before
  the instance cell is initialized (early startup, unchanged behavior).
- The store's bucket handlers (make/get_info/list/delete + the
  table-bucket delete guard and emptiness check) use the per-context
  variants and this instance's disks.

Acceptance (e2e): two embedded servers with different credentials are now
isolated end to end — each authenticates only its own key, neither sees
the other's buckets or objects, and both data planes stay intact. The
embedded module doc drops the shared-IAM caveat.

579 ecstore bucket/metadata/peer regressions plus the embedded basic and
deferred-IAM e2e stay green.

crates/ecstore/src/store/bucket.rs

@@ -40,8 +40,8 @@ fn validate_table_bucket_delete_allowed(
     Ok(())
 }
 
-async fn table_catalog_metadata_exists(bucket: &str) -> bool {
-    let local_disks = all_local_disk().await;
+async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> bool {
+    let local_disks = runtime_sources::local_disks_in(ctx).await;
     for disk in local_disks.iter() {
         let catalog_path = disk.path().join(bucket).join(BUCKET_TABLE_RESERVED_PREFIX);
         if has_xlmeta_files(&catalog_path).await {
@@ -52,12 +52,12 @@ async fn table_catalog_metadata_exists(bucket: &str) -> bool {
     false
 }
 
-async fn validate_table_bucket_delete_guard(bucket: &str) -> Result<()> {
-    let table_bucket_enabled = metadata_sys::get(bucket)
+async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<()> {
+    let table_bucket_enabled = metadata_sys::get_in(ctx, bucket)
         .await
         .is_ok_and(|metadata| metadata.table_bucket_enabled());
     if table_bucket_enabled {
-        validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(bucket).await)?;
+        validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(ctx, bucket).await)?;
     }
 
     Ok(())
@@ -106,7 +106,7 @@ impl ECStore {
             self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()).await?;
         }
 
-        metadata_sys::remove_bucket_metadata(bucket).await?;
+        metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket).await?;
         runtime_sources::delete_bucket_monitor_entry(bucket);
         Ok(())
     }
@@ -186,9 +186,7 @@ impl ECStore {
             meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
         }
 
-        meta.save().await?;
-
-        set_bucket_metadata(bucket.to_string(), meta).await?;
+        metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await?;
 
         Ok(())
     }
@@ -197,7 +195,7 @@ impl ECStore {
     pub(super) async fn handle_get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
         let mut info = self.peer_sys.get_bucket_info(bucket, opts).await?;
 
-        if let Ok(sys) = metadata_sys::get(bucket).await {
+        if let Ok(sys) = metadata_sys::get_in(&self.ctx, bucket).await {
             if should_override_created_from_metadata(sys.created) {
                 info.created = Some(sys.created);
             }
@@ -216,7 +214,7 @@ impl ECStore {
 
         if !opts.no_metadata {
             for bucket in buckets.iter_mut() {
-                if let Ok(created) = metadata_sys::created_at(&bucket.name).await
+                if let Ok(created) = metadata_sys::created_at_in(&self.ctx, &bucket.name).await
                     && should_override_created_from_metadata(created)
                 {
                     bucket.created = Some(created);
@@ -270,7 +268,7 @@ impl ECStore {
             return Err(to_object_err(storage_err, vec![bucket]));
         }
 
-        validate_table_bucket_delete_guard(bucket).await?;
+        validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
 
         let sr_mark_delete = opts.srdelete_op == SRBucketDeleteOp::MarkDelete;
         let sr_purge = opts.srdelete_op == SRBucketDeleteOp::Purge;
@@ -280,7 +278,7 @@ impl ECStore {
         // is not set, return BucketNotEmpty error.
         // Note: Empty directories (left after object deletion) should NOT count as objects.
         if !opts.force && !sr_mark_delete {
-            let local_disks = all_local_disk().await;
+            let local_disks = runtime_sources::local_disks_in(&self.ctx).await;
             for disk in local_disks.iter() {
                 // Check if bucket directory contains any xl.meta files (actual objects)
                 // We recursively scan for xl.meta files to determine if bucket has objects

crates/ecstore/src/store/init.rs

@@ -339,7 +339,7 @@ impl ECStore {
             runtime_sources::record_local_disks(&instance_ctx, local_disks).await;
         }
 
-        let peer_sys = S3PeerSys::new(&endpoint_pools);
+        let peer_sys = S3PeerSys::new_with_instance_ctx(&endpoint_pools, instance_ctx.clone());
         let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
         pool_meta.dont_save = true;
 

crates/ecstore/src/store/mod.rs

@@ -20,7 +20,7 @@ use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
 use crate::bucket::lifecycle::bucket_lifecycle_ops::{
     enqueue_immediate_expiry, enqueue_transition_immediate, init_background_expiry,
 };
-use crate::bucket::metadata_sys::{self, set_bucket_metadata};
+use crate::bucket::metadata_sys;
 use crate::bucket::utils::check_abort_multipart_args;
 use crate::bucket::utils::check_complete_multipart_args;
 use crate::bucket::utils::check_copy_obj_args;
@@ -865,7 +865,7 @@ mod tests {
             id: uuid::Uuid::new_v4(),
             disk_map: std::collections::HashMap::new(),
             pools: Vec::new(),
-            peer_sys: crate::cluster::rpc::S3PeerSys::new(&endpoint_pools),
+            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
             pool_meta: RwLock::new(PoolMeta::default()),
             rebalance_meta: RwLock::new(None),
             decommission_cancelers: RwLock::new(Vec::new()),