fix(s3): resume in-flight GET streams after rebalance relocation (#5791)

* fix(s3): resume in-flight GET streams after rebalance relocation

* fix(s3): address GET resume review findings

---------

Co-authored-by: zhengsf <zhengsf@kaopucloud.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <cxymds@gmail.com>
Co-authored-by: 马登山 <cxymds@qq.com>

.config/nextest.toml

@@ -55,6 +55,13 @@ test-group = 'ecstore-serial-flaky'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
+# The production-handler relocation regression builds an isolated 8-disk,
+# 2-pool store and commits a 72 MiB multipart object. Keep that cross-disk IO
+# from overlapping the ecstore commit fixtures above.
+[[profile.default.overrides]]
+filter = 'package(rustfs) & test(execute_get_object_resumes_from_relocated_pool_without_splicing_body)'
+test-group = 'ecstore-serial-flaky'
+
 # Embedded integration-test binaries discover an ephemeral port and release
 # the probe listener before RustFS binds it. Serialize that cross-process
 # TOCTOU window; retries would only hide real startup failures.
@@ -151,6 +158,10 @@ test-group = 'e2e-reliability'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
+[[profile.ci.overrides]]
+filter = 'package(rustfs) & test(execute_get_object_resumes_from_relocated_pool_without_splicing_body)'
+test-group = 'ecstore-serial-flaky'
+
 # Match the default-profile embedded test isolation without quarantining or
 # retrying failures in CI.
 [[profile.ci.overrides]]

rustfs/src/app/gating_test_env.rs

@@ -100,3 +100,63 @@ pub(crate) async fn shared_gating_ecstore() -> Arc<ECStore> {
     let _ = SHARED_GATING_ENV.set((disk_paths, ecstore.clone(), temp_dir));
     ecstore
 }
+
+/// Like [`shared_gating_ecstore`], but also returns the backing disk paths so
+/// tests can remove on-disk shards and simulate the object data vanishing
+/// mid-stream.
+pub(crate) async fn shared_gating_ecstore_and_disk_paths() -> (Vec<PathBuf>, Arc<ECStore>) {
+    let _ = shared_gating_ecstore().await;
+    let (disk_paths, store, _) = SHARED_GATING_ENV.get().expect("gating env must be initialized");
+    (disk_paths.clone(), store.clone())
+}
+
+/// Build an isolated two-pool store for tests that move object data between
+/// pools while a production read is in flight.
+pub(crate) async fn isolated_multi_pool_ecstore() -> (TempDir, Vec<Vec<PathBuf>>, Arc<ECStore>) {
+    let temp_dir = TempDir::new().expect("create temp dir for multi-pool gating test");
+    let mut pool_disk_paths = Vec::with_capacity(2);
+    let mut pools = Vec::with_capacity(2);
+    for pool_index in 0..2 {
+        let mut disk_paths = Vec::with_capacity(4);
+        let mut endpoints = Vec::with_capacity(4);
+        for disk_index in 0..4 {
+            let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
+            fs::create_dir_all(&disk_path)
+                .await
+                .expect("create multi-pool gating test disk");
+            let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("test disk path must be utf8"))
+                .expect("multi-pool test endpoint must parse");
+            endpoint.set_pool_index(pool_index);
+            endpoint.set_set_index(0);
+            endpoint.set_disk_index(disk_index);
+            endpoints.push(endpoint);
+            disk_paths.push(disk_path);
+        }
+        pool_disk_paths.push(disk_paths);
+        pools.push(PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 4,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: format!("multi-pool-gating-{pool_index}"),
+            platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+        });
+    }
+
+    let endpoint_pools = EndpointServerPools(pools);
+    let instance_ctx = super::storage_api::test::runtime::new_instance_ctx();
+    super::storage_api::test::runtime::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
+        .await
+        .expect("initialize isolated multi-pool disks");
+    let store = ECStore::new_with_instance_ctx(
+        "127.0.0.1:0".parse().expect("multi-pool test address must parse"),
+        endpoint_pools,
+        CancellationToken::new(),
+        instance_ctx,
+    )
+    .await
+    .expect("initialize isolated multi-pool store");
+    metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+    (temp_dir, pool_disk_paths, store)
+}

rustfs/src/app/storage_api.rs

@@ -214,6 +214,19 @@ pub(crate) mod runtime {
     ) -> Result<(), crate::storage::storage_api::StorageError> {
         crate::storage::storage_api::init_local_disks(endpoint_pools).await
     }
+
+    #[cfg(test)]
+    pub(crate) fn new_instance_ctx() -> Arc<crate::storage::storage_api::InstanceContext> {
+        crate::storage::storage_api::new_instance_ctx()
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn init_local_disks_with_instance_ctx(
+        instance_ctx: &Arc<crate::storage::storage_api::InstanceContext>,
+        endpoint_pools: crate::storage::storage_api::EndpointServerPools,
+    ) -> Result<(), crate::storage::storage_api::StorageError> {
+        crate::storage::storage_api::init_local_disks_with_instance_ctx(instance_ctx, endpoint_pools).await
+    }
 }
 
 pub(crate) mod runtime_sources {

scripts/check_logging_guardrails.sh

@@ -857,6 +857,11 @@ if rg -n -U '(info|warn)!\(\s*target: "rustfs::heal::manager",[\s\S]{0,1000}"Hea
   exit 1
 fi
 
+if rg -n -U 'info!\([\s\S]{0,1000}"GetObject streaming body resumed from a reopened object read"' rustfs/src/app/object_usecase.rs >/dev/null; then
+  echo "❌ logging guardrail violation: successful per-object GetObject resume events must stay below INFO" >&2
+  exit 1
+fi
+
 demoted_task_sites="$(rg -c -F 'demote_to_debug_when!(self.heal_type.is_per_object()' crates/heal/src/heal/task.rs || echo 0)"
 if [[ "$demoted_task_sites" -lt 4 ]]; then
   echo "❌ logging guardrail violation: per-object heal task lifecycle/failure logs must stay demoted to DEBUG via demote_to_debug_when! (expected >= 4 sites in crates/heal/src/heal/task.rs, found $demoted_task_sites)" >&2