fix(ci): keep PR e2e smoke lane from timing out (#5649)

fix(ci): prevent e2e smoke lane timeout

.config/nextest.toml

@@ -212,6 +212,17 @@ default-filter = """
 """
 fail-fast = false
 
+[profile.e2e-smoke.junit]
+path = "junit.xml"
+
+# The pagination boundary cases can stall when a server/listing regression
+# prevents the continuation request from completing. Keep the timeout scoped
+# to those known failure modes so legitimate lifecycle/tiering waits retain
+# their test-level timing budget.
+[[profile.e2e-smoke.overrides]]
+filter = 'package(e2e_test) & test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/)'
+slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
+
 # ---------------------------------------------------------------------------
 # e2e-repl-nightly profile — scheduled full replication e2e lane (repl-1)
 # ---------------------------------------------------------------------------

.github/workflows/ci.yml

@@ -340,9 +340,11 @@ jobs:
       - name: Annotate early-stop reason
         if: failure() && github.event_name == 'pull_request'
         run: |
-          echo "## CI early-stop" >> "$GITHUB_STEP_SUMMARY"
+          {
-          echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners." >> "$GITHUB_STEP_SUMMARY"
+            echo "## CI early-stop"
-          echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure." >> "$GITHUB_STEP_SUMMARY"
+            echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners."
+            echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure."
+          } >> "$GITHUB_STEP_SUMMARY"
 
       # curl rather than `gh`: every existing `gh` call in this repo runs on
       # ubuntu-latest, and the sm-standard-* images are custom and trimmed (they
@@ -665,15 +667,17 @@ jobs:
       - name: Make binary executable
         run: chmod +x ./target/debug/rustfs
 
-      # Guard the security negative-auth smoke subset (backlog#1151 sec-5)
+      # Build the e2e test graph once. The archive is reused by the security
-      # against a rename or deletion silently dropping it out of the e2e-smoke
+      # count-floor check and the smoke run below, avoiding a second compile of
-      # filter. The script lists what the profile selects and fails if the count
+      # the same e2e_test target on cold runners (backlog#1645).
-      # of security auth-rejection tests falls below the committed floor in
+      - name: Archive e2e smoke test binaries
-      # .config/security-smoke-floor.txt (infra-12 count-floor mechanism). Run
+        env:
-      # before the smoke suite so a thinned gate fails fast; the `nextest list`
+          NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
-      # here compiles the e2e_test binaries the run below reuses.
+          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
-      - name: Check security smoke subset count floor
+        run: |
-        run: ./scripts/check_security_smoke_count.sh check
+          cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
+          cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
+          ./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
 
       # PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
       # profile.e2e-smoke default-filter in .config/nextest.toml is the single
@@ -681,7 +685,30 @@ jobs:
       # adding new e2e jobs here. Each test spawns its own rustfs server on a
       # random port and reuses the downloaded debug binary above.
       - name: Run e2e smoke suite
-        run: cargo nextest run --profile e2e-smoke -p e2e_test
+        env:
+          NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
+        run: |
+          cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
+            --status-level all --final-status-level all --failure-output final
+
+      - name: Upload e2e smoke diagnostics
+        if: failure()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: e2e-smoke-diagnostics-${{ github.run_number }}
+          path: |
+            ${{ runner.temp }}/rustfs-e2e-smoke-logs/
+            ${{ runner.temp }}/rustfs-e2e-smoke-list.json
+          if-no-files-found: warn
+
+      - name: Upload e2e smoke JUnit report
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: e2e-smoke-junit-${{ github.run_number }}
+          path: target/nextest/e2e-smoke/junit.xml
+          if-no-files-found: warn
 
       - name: Install s3s-e2e test tool
         uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2

crates/e2e_test/src/common.rs

@@ -50,6 +50,21 @@ pub const ENV_RUSTFS_BUILD_FEATURES: &str = "RUSTFS_BUILD_FEATURES";
 pub const TEST_BUCKET: &str = "e2e-test-bucket";
 const RUSTFS_FULL_FEATURE: &str = "full";
 
+fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
+    let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
+    Some(log_dir.join(format!("{temp_name}.log")))
+}
+
+fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
+    let log_dir = std::env::var_os("RUSTFS_E2E_LOG_DIR")?;
+    if stdfs::create_dir_all(&log_dir).is_err() {
+        warn!(?log_dir, "failed to create configured E2E server log directory");
+        return None;
+    }
+
+    capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
+}
+
 fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config {
     let credentials = Credentials::new(access_key, secret_key, None, None, provider_name);
     let mut config = Config::builder()
@@ -361,6 +376,7 @@ impl RustFSTestEnvironment {
     pub async fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
         let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
         fs::create_dir_all(&temp_dir).await?;
+        let capture_log_path = configured_capture_log_path(&temp_dir);
 
         // Use a unique port for each test environment
         let port = Self::find_available_port().await?;
@@ -374,7 +390,7 @@ impl RustFSTestEnvironment {
             access_key: DEFAULT_ACCESS_KEY.to_string(),
             secret_key: DEFAULT_SECRET_KEY.to_string(),
             process: None,
-            capture_log_path: None,
+            capture_log_path,
         })
     }
 
@@ -382,6 +398,7 @@ impl RustFSTestEnvironment {
     pub async fn with_address(address: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
         let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
         fs::create_dir_all(&temp_dir).await?;
+        let capture_log_path = configured_capture_log_path(&temp_dir);
 
         let url = format!("http://{address}");
 
@@ -392,7 +409,7 @@ impl RustFSTestEnvironment {
             access_key: DEFAULT_ACCESS_KEY.to_string(),
             secret_key: DEFAULT_SECRET_KEY.to_string(),
             process: None,
-            capture_log_path: None,
+            capture_log_path,
         })
     }
 
@@ -1392,6 +1409,14 @@ mod tests {
         assert_eq!(normalize_rustfs_build_features(" , "), None);
     }
 
+    #[test]
+    fn capture_log_path_uses_temp_directory_basename() {
+        assert_eq!(
+            capture_log_path(Path::new("/tmp/e2e-logs"), "/tmp/rustfs_e2e_test_abc"),
+            Some(PathBuf::from("/tmp/e2e-logs/rustfs_e2e_test_abc.log"))
+        );
+    }
+
     #[test]
     fn full_feature_enables_any_required_feature() {
         assert!(rustfs_build_feature_enabled(Some("full"), "sftp"));

scripts/check_security_smoke_count.sh

@@ -30,8 +30,9 @@
 # Keep this regex in sync with the module names in the e2e-smoke filter.
 #
 # Usage:
-#   scripts/check_security_smoke_count.sh          # count check (alias of check)
+#   scripts/check_security_smoke_count.sh                    # count check (alias of check)
-#   scripts/check_security_smoke_count.sh check    # count check only
+#   scripts/check_security_smoke_count.sh check              # count check only
+#   scripts/check_security_smoke_count.sh check listing.json  # reuse a nextest listing
 
 set -euo pipefail
 
@@ -51,6 +52,17 @@ case "$mode" in
         ;;
 esac
 
+if (( $# > 2 )); then
+    echo "usage: $0 [check] [nextest-listing.json]" >&2
+    exit 2
+fi
+
+listing_file="${2:-}"
+if [[ -n "$listing_file" && ! -f "$listing_file" ]]; then
+    echo "error: nextest listing does not exist: $listing_file" >&2
+    exit 1
+fi
+
 floor="$(grep -Ev '^[[:space:]]*(#|$)' "$FLOOR_FILE" | head -n1 | tr -d '[:space:]')"
 if ! [[ "$floor" =~ ^[0-9]+$ ]]; then
     echo "error: $FLOOR_FILE does not contain a numeric floor (got: '$floor')" >&2
@@ -59,13 +71,20 @@ fi
 
 # List via the e2e-smoke PROFILE so the default-filter is applied: only the
 # tests the PR smoke suite would actually run appear in the output. Count the
-# ones whose test name starts with a security module prefix. The structured JSON
+# ones whose test name starts with a security module prefix. CI passes the
-# listing is used (not the human format) for the same reason infra-12 does: the
+# listing produced from the smoke archive so this guard does not rebuild the
-# human format's indentation varies across nextest versions; a JSON schema change
+# e2e test graph immediately before the smoke run. The structured JSON listing
-# makes jq fail loudly rather than silently collapsing the count to zero.
+# is used (not the human format) because a schema change makes jq fail loudly
-count="$(cargo nextest list --profile e2e-smoke -p e2e_test --message-format json \
+# rather than silently collapsing the count to zero.
-    | jq --arg re "$SECURITY_SMOKE_REGEX" \
+if [[ -n "$listing_file" ]]; then
-        '[."rust-suites"[].testcases | to_entries[] | select(.key | test($re))] | length')"
+    count="$(jq --arg re "$SECURITY_SMOKE_REGEX" \
+        '[."rust-suites"[].testcases | to_entries[] | select(.value["filter-match"].status == "matches") | select(.key | test($re))] | length' \
+        "$listing_file")"
+else
+    count="$(cargo nextest list --profile e2e-smoke -p e2e_test --message-format json \
+        | jq --arg re "$SECURITY_SMOKE_REGEX" \
+            '[."rust-suites"[].testcases | to_entries[] | select(.value["filter-match"].status == "matches") | select(.key | test($re))] | length')"
+fi
 if ! [[ "$count" =~ ^[0-9]+$ ]]; then
     echo "error: could not parse nextest JSON listing (got count: '$count')" >&2
     exit 1