diff --git a/.config/nextest.toml b/.config/nextest.toml index 1873e05bc..9e915404a 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -212,6 +212,17 @@ default-filter = """ """ fail-fast = false +[profile.e2e-smoke.junit] +path = "junit.xml" + +# The pagination boundary cases can stall when a server/listing regression +# prevents the continuation request from completing. Keep the timeout scoped +# to those known failure modes so legitimate lifecycle/tiering waits retain +# their test-level timing budget. +[[profile.e2e-smoke.overrides]] +filter = 'package(e2e_test) & test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/)' +slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" } + # --------------------------------------------------------------------------- # e2e-repl-nightly profile — scheduled full replication e2e lane (repl-1) # --------------------------------------------------------------------------- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fa68b682..dfd2afd97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -340,9 +340,11 @@ jobs: - name: Annotate early-stop reason if: failure() && github.event_name == 'pull_request' run: | - echo "## CI early-stop" >> "$GITHUB_STEP_SUMMARY" - echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners." >> "$GITHUB_STEP_SUMMARY" - echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure." >> "$GITHUB_STEP_SUMMARY" + { + echo "## CI early-stop" + echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners." + echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure." + } >> "$GITHUB_STEP_SUMMARY" # curl rather than `gh`: every existing `gh` call in this repo runs on # ubuntu-latest, and the sm-standard-* images are custom and trimmed (they @@ -665,15 +667,17 @@ jobs: - name: Make binary executable run: chmod +x ./target/debug/rustfs - # Guard the security negative-auth smoke subset (backlog#1151 sec-5) - # against a rename or deletion silently dropping it out of the e2e-smoke - # filter. The script lists what the profile selects and fails if the count - # of security auth-rejection tests falls below the committed floor in - # .config/security-smoke-floor.txt (infra-12 count-floor mechanism). Run - # before the smoke suite so a thinned gate fails fast; the `nextest list` - # here compiles the e2e_test binaries the run below reuses. - - name: Check security smoke subset count floor - run: ./scripts/check_security_smoke_count.sh check + # Build the e2e test graph once. The archive is reused by the security + # count-floor check and the smoke run below, avoiding a second compile of + # the same e2e_test target on cold runners (backlog#1645). + - name: Archive e2e smoke test binaries + env: + NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst + NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json + run: | + cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}" + cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}" + ./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}" # PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The # profile.e2e-smoke default-filter in .config/nextest.toml is the single @@ -681,7 +685,30 @@ jobs: # adding new e2e jobs here. Each test spawns its own rustfs server on a # random port and reuses the downloaded debug binary above. - name: Run e2e smoke suite - run: cargo nextest run --profile e2e-smoke -p e2e_test + env: + NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst + RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs + run: | + cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \ + --status-level all --final-status-level all --failure-output final + + - name: Upload e2e smoke diagnostics + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: e2e-smoke-diagnostics-${{ github.run_number }} + path: | + ${{ runner.temp }}/rustfs-e2e-smoke-logs/ + ${{ runner.temp }}/rustfs-e2e-smoke-list.json + if-no-files-found: warn + + - name: Upload e2e smoke JUnit report + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: e2e-smoke-junit-${{ github.run_number }} + path: target/nextest/e2e-smoke/junit.xml + if-no-files-found: warn - name: Install s3s-e2e test tool uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2 diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index 0c4c4416f..92b5510eb 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -50,6 +50,21 @@ pub const ENV_RUSTFS_BUILD_FEATURES: &str = "RUSTFS_BUILD_FEATURES"; pub const TEST_BUCKET: &str = "e2e-test-bucket"; const RUSTFS_FULL_FEATURE: &str = "full"; +fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option { + let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy(); + Some(log_dir.join(format!("{temp_name}.log"))) +} + +fn configured_capture_log_path(temp_dir: &str) -> Option { + let log_dir = std::env::var_os("RUSTFS_E2E_LOG_DIR")?; + if stdfs::create_dir_all(&log_dir).is_err() { + warn!(?log_dir, "failed to create configured E2E server log directory"); + return None; + } + + capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned()) +} + fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config { let credentials = Credentials::new(access_key, secret_key, None, None, provider_name); let mut config = Config::builder() @@ -361,6 +376,7 @@ impl RustFSTestEnvironment { pub async fn new() -> Result> { let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4()); fs::create_dir_all(&temp_dir).await?; + let capture_log_path = configured_capture_log_path(&temp_dir); // Use a unique port for each test environment let port = Self::find_available_port().await?; @@ -374,7 +390,7 @@ impl RustFSTestEnvironment { access_key: DEFAULT_ACCESS_KEY.to_string(), secret_key: DEFAULT_SECRET_KEY.to_string(), process: None, - capture_log_path: None, + capture_log_path, }) } @@ -382,6 +398,7 @@ impl RustFSTestEnvironment { pub async fn with_address(address: &str) -> Result> { let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4()); fs::create_dir_all(&temp_dir).await?; + let capture_log_path = configured_capture_log_path(&temp_dir); let url = format!("http://{address}"); @@ -392,7 +409,7 @@ impl RustFSTestEnvironment { access_key: DEFAULT_ACCESS_KEY.to_string(), secret_key: DEFAULT_SECRET_KEY.to_string(), process: None, - capture_log_path: None, + capture_log_path, }) } @@ -1392,6 +1409,14 @@ mod tests { assert_eq!(normalize_rustfs_build_features(" , "), None); } + #[test] + fn capture_log_path_uses_temp_directory_basename() { + assert_eq!( + capture_log_path(Path::new("/tmp/e2e-logs"), "/tmp/rustfs_e2e_test_abc"), + Some(PathBuf::from("/tmp/e2e-logs/rustfs_e2e_test_abc.log")) + ); + } + #[test] fn full_feature_enables_any_required_feature() { assert!(rustfs_build_feature_enabled(Some("full"), "sftp")); diff --git a/scripts/check_security_smoke_count.sh b/scripts/check_security_smoke_count.sh index aa5f6269d..190dfb6d8 100755 --- a/scripts/check_security_smoke_count.sh +++ b/scripts/check_security_smoke_count.sh @@ -30,8 +30,9 @@ # Keep this regex in sync with the module names in the e2e-smoke filter. # # Usage: -# scripts/check_security_smoke_count.sh # count check (alias of check) -# scripts/check_security_smoke_count.sh check # count check only +# scripts/check_security_smoke_count.sh # count check (alias of check) +# scripts/check_security_smoke_count.sh check # count check only +# scripts/check_security_smoke_count.sh check listing.json # reuse a nextest listing set -euo pipefail @@ -51,6 +52,17 @@ case "$mode" in ;; esac +if (( $# > 2 )); then + echo "usage: $0 [check] [nextest-listing.json]" >&2 + exit 2 +fi + +listing_file="${2:-}" +if [[ -n "$listing_file" && ! -f "$listing_file" ]]; then + echo "error: nextest listing does not exist: $listing_file" >&2 + exit 1 +fi + floor="$(grep -Ev '^[[:space:]]*(#|$)' "$FLOOR_FILE" | head -n1 | tr -d '[:space:]')" if ! [[ "$floor" =~ ^[0-9]+$ ]]; then echo "error: $FLOOR_FILE does not contain a numeric floor (got: '$floor')" >&2 @@ -59,13 +71,20 @@ fi # List via the e2e-smoke PROFILE so the default-filter is applied: only the # tests the PR smoke suite would actually run appear in the output. Count the -# ones whose test name starts with a security module prefix. The structured JSON -# listing is used (not the human format) for the same reason infra-12 does: the -# human format's indentation varies across nextest versions; a JSON schema change -# makes jq fail loudly rather than silently collapsing the count to zero. -count="$(cargo nextest list --profile e2e-smoke -p e2e_test --message-format json \ - | jq --arg re "$SECURITY_SMOKE_REGEX" \ - '[."rust-suites"[].testcases | to_entries[] | select(.key | test($re))] | length')" +# ones whose test name starts with a security module prefix. CI passes the +# listing produced from the smoke archive so this guard does not rebuild the +# e2e test graph immediately before the smoke run. The structured JSON listing +# is used (not the human format) because a schema change makes jq fail loudly +# rather than silently collapsing the count to zero. +if [[ -n "$listing_file" ]]; then + count="$(jq --arg re "$SECURITY_SMOKE_REGEX" \ + '[."rust-suites"[].testcases | to_entries[] | select(.value["filter-match"].status == "matches") | select(.key | test($re))] | length' \ + "$listing_file")" +else + count="$(cargo nextest list --profile e2e-smoke -p e2e_test --message-format json \ + | jq --arg re "$SECURITY_SMOKE_REGEX" \ + '[."rust-suites"[].testcases | to_entries[] | select(.value["filter-match"].status == "matches") | select(.key | test($re))] | length')" +fi if ! [[ "$count" =~ ^[0-9]+$ ]]; then echo "error: could not parse nextest JSON listing (got count: '$count')" >&2 exit 1