fix(s3): resume in-flight GET streams after rebalance relocation (#5791)

* fix(s3): resume in-flight GET streams after rebalance relocation

* fix(s3): address GET resume review findings

---------

Co-authored-by: zhengsf <zhengsf@kaopucloud.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <cxymds@gmail.com>
Co-authored-by: 马登山 <cxymds@qq.com>

rustfs/src/app/gating_test_env.rs

@@ -100,3 +100,63 @@ pub(crate) async fn shared_gating_ecstore() -> Arc<ECStore> {
     let _ = SHARED_GATING_ENV.set((disk_paths, ecstore.clone(), temp_dir));
     ecstore
 }
+
+/// Like [`shared_gating_ecstore`], but also returns the backing disk paths so
+/// tests can remove on-disk shards and simulate the object data vanishing
+/// mid-stream.
+pub(crate) async fn shared_gating_ecstore_and_disk_paths() -> (Vec<PathBuf>, Arc<ECStore>) {
+    let _ = shared_gating_ecstore().await;
+    let (disk_paths, store, _) = SHARED_GATING_ENV.get().expect("gating env must be initialized");
+    (disk_paths.clone(), store.clone())
+}
+
+/// Build an isolated two-pool store for tests that move object data between
+/// pools while a production read is in flight.
+pub(crate) async fn isolated_multi_pool_ecstore() -> (TempDir, Vec<Vec<PathBuf>>, Arc<ECStore>) {
+    let temp_dir = TempDir::new().expect("create temp dir for multi-pool gating test");
+    let mut pool_disk_paths = Vec::with_capacity(2);
+    let mut pools = Vec::with_capacity(2);
+    for pool_index in 0..2 {
+        let mut disk_paths = Vec::with_capacity(4);
+        let mut endpoints = Vec::with_capacity(4);
+        for disk_index in 0..4 {
+            let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
+            fs::create_dir_all(&disk_path)
+                .await
+                .expect("create multi-pool gating test disk");
+            let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("test disk path must be utf8"))
+                .expect("multi-pool test endpoint must parse");
+            endpoint.set_pool_index(pool_index);
+            endpoint.set_set_index(0);
+            endpoint.set_disk_index(disk_index);
+            endpoints.push(endpoint);
+            disk_paths.push(disk_path);
+        }
+        pool_disk_paths.push(disk_paths);
+        pools.push(PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 4,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: format!("multi-pool-gating-{pool_index}"),
+            platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+        });
+    }
+
+    let endpoint_pools = EndpointServerPools(pools);
+    let instance_ctx = super::storage_api::test::runtime::new_instance_ctx();
+    super::storage_api::test::runtime::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
+        .await
+        .expect("initialize isolated multi-pool disks");
+    let store = ECStore::new_with_instance_ctx(
+        "127.0.0.1:0".parse().expect("multi-pool test address must parse"),
+        endpoint_pools,
+        CancellationToken::new(),
+        instance_ctx,
+    )
+    .await
+    .expect("initialize isolated multi-pool store");
+    metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+    (temp_dir, pool_disk_paths, store)
+}

rustfs/src/app/storage_api.rs

@@ -214,6 +214,19 @@ pub(crate) mod runtime {
     ) -> Result<(), crate::storage::storage_api::StorageError> {
         crate::storage::storage_api::init_local_disks(endpoint_pools).await
     }
+
+    #[cfg(test)]
+    pub(crate) fn new_instance_ctx() -> Arc<crate::storage::storage_api::InstanceContext> {
+        crate::storage::storage_api::new_instance_ctx()
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn init_local_disks_with_instance_ctx(
+        instance_ctx: &Arc<crate::storage::storage_api::InstanceContext>,
+        endpoint_pools: crate::storage::storage_api::EndpointServerPools,
+    ) -> Result<(), crate::storage::storage_api::StorageError> {
+        crate::storage::storage_api::init_local_disks_with_instance_ctx(instance_ctx, endpoint_pools).await
+    }
 }
 
 pub(crate) mod runtime_sources {