fix(ecstore): stop scan_dir emitting entries past a limit hit inside a subdirectory (#7049)

* fix(ecstore): stop scan_dir emitting entries past a limit hit inside a subdirectory

scan_dir's flush loop recurses into a pending subdirectory when the
current sibling entry's page limit is reached mid-recursion, but kept
writing the current (later-sorting) entry regardless. gather_results
then builds the next page's continuation marker from that later entry,
which permanently skips the still-unscanned tail of the subdirectory
on resume instead of just deferring it to the next page.

Add a limit re-check right after the flush loop, before the current
entry is written, so scan_dir stops cleanly at the true last-written
key. Reproduces and fixes the rc.5 recursive ListObjectsV2 data-loss
report (7826/7881 keys, contiguous 55-key block silently dropped).

Adds scan_dir_does_not_emit_entries_past_a_limit_hit_inside_a_subdirectory.

* fix(ecstore): re-check the page limit on every dir_stack flush iteration

The flush loop that drains dir_stack can pop and recurse into more than
one pending subdirectory per outer iteration (whenever more than one
stack entry sorts below the current sibling entry). The limit re-check
added in the previous commit only ran once, after the whole flush loop
exited - so if the first recursive scan_dir call already exhausted the
page limit, the loop's next pop+recurse still went ahead and scanned
(and emitted entries for) another subdirectory beyond where the page
was supposed to stop.

Confirmed against production data: a bucket with ~1.17M objects under
one prefix still cut a recursive ListObjectsV2 listing short (825 of an
expected much larger next page, IsTruncated=false) even with the first
fix deployed, at a two-level-nested subdirectory. Move the check inside
the while loop so it runs before every pop, not just once after.

---------

Co-authored-by: Claude Agent <agent@local>

rustfs/src/app/gating_test_env.rs

@@ -29,6 +29,7 @@ use super::storage_api::test::contract::bucket::MakeBucketOptions;
 use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions};
 use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
 use super::{context::AppContext, object_traffic_health::ObjectTrafficHealth};
+use std::future::Future;
 use std::path::PathBuf;
 use std::sync::{Arc, OnceLock};
 use tempfile::TempDir;
@@ -38,6 +39,27 @@ use tokio_util::sync::CancellationToken;
 static SHARED_GATING_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>, TempDir)> = OnceLock::new();
 static SHARED_GATING_INIT: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
 
+pub(crate) fn run_large_stack_test<F, Fut>(name: &'static str, test: F)
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: Future<Output = ()> + 'static,
+{
+    std::thread::Builder::new()
+        .name(name.to_string())
+        .stack_size(32 * 1024 * 1024)
+        .spawn(move || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("large-stack test runtime should build");
+
+            runtime.block_on(test());
+        })
+        .expect("large-stack test thread should spawn")
+        .join()
+        .expect("large-stack test thread should finish");
+}
+
 /// Return a shared 4-disk `ECStore` with bucket metadata initialized.
 ///
 /// The first caller creates the store and initializes the metadata system;

rustfs/src/app/object/head.rs

@@ -1003,9 +1003,16 @@ mod tests {
     /// runtime configured as `head = local_only`: any HEAD that wrongly
     /// enters the runtime shows up as a `filtered` count, so a zero counter
     /// proves the gate held.
-    #[tokio::test]
+    #[test]
     #[serial_test::serial]
-    async fn execute_head_object_odm_gate_against_real_store() {
+    fn execute_head_object_odm_gate_against_real_store() {
+        crate::app::gating_test_env::run_large_stack_test(
+            "execute-head-object-odm-gate",
+            execute_head_object_odm_gate_against_real_store_inner,
+        );
+    }
+
+    async fn execute_head_object_odm_gate_against_real_store_inner() {
         use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
 
         let store = crate::app::gating_test_env::shared_gating_ecstore().await;

rustfs/src/app/object/internal_put.rs

@@ -829,9 +829,16 @@ mod tests {
         assert_eq!(err.code, S3ErrorCode::InvalidRequest);
     }
 
-    #[tokio::test]
+    #[test]
     #[serial_test::serial]
-    async fn internal_multipart_roundtrip_completes_and_abort_leaves_nothing() {
+    fn internal_multipart_roundtrip_completes_and_abort_leaves_nothing() {
+        crate::app::gating_test_env::run_large_stack_test(
+            "internal-multipart-roundtrip",
+            internal_multipart_roundtrip_completes_and_abort_leaves_nothing_inner,
+        );
+    }
+
+    async fn internal_multipart_roundtrip_completes_and_abort_leaves_nothing_inner() {
         const FIRST_PART_SIZE: usize = 5 * 1024 * 1024;
         let (store, bucket) = internal_put_test_bucket("internal-mpu").await;
         let usecase = DefaultObjectUsecase::from_global();