mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-05 12:57:42 +00:00
fix(ci): keep PR e2e smoke lane from timing out (#5649)
fix(ci): prevent e2e smoke lane timeout
This commit is contained in:
cxymds
@@ -212,6 +212,17 @@ default-filter = """
|
||||
"""
|
||||
fail-fast = false
|
||||
|
||||
[profile.e2e-smoke.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
# The pagination boundary cases can stall when a server/listing regression
|
||||
# prevents the continuation request from completing. Keep the timeout scoped
|
||||
# to those known failure modes so legitimate lifecycle/tiering waits retain
|
||||
# their test-level timing budget.
|
||||
[[profile.e2e-smoke.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/)'
|
||||
slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-repl-nightly profile — scheduled full replication e2e lane (repl-1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+40
-13
@@ -340,9 +340,11 @@ jobs:
|
||||
- name: Annotate early-stop reason
|
||||
if: failure() && github.event_name == 'pull_request'
|
||||
run: |
|
||||
echo "## CI early-stop" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners." >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure." >> "$GITHUB_STEP_SUMMARY"
|
||||
{
|
||||
echo "## CI early-stop"
|
||||
echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners."
|
||||
echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# curl rather than `gh`: every existing `gh` call in this repo runs on
|
||||
# ubuntu-latest, and the sm-standard-* images are custom and trimmed (they
|
||||
@@ -665,15 +667,17 @@ jobs:
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
# Guard the security negative-auth smoke subset (backlog#1151 sec-5)
|
||||
# against a rename or deletion silently dropping it out of the e2e-smoke
|
||||
# filter. The script lists what the profile selects and fails if the count
|
||||
# of security auth-rejection tests falls below the committed floor in
|
||||
# .config/security-smoke-floor.txt (infra-12 count-floor mechanism). Run
|
||||
# before the smoke suite so a thinned gate fails fast; the `nextest list`
|
||||
# here compiles the e2e_test binaries the run below reuses.
|
||||
- name: Check security smoke subset count floor
|
||||
run: ./scripts/check_security_smoke_count.sh check
|
||||
# Build the e2e test graph once. The archive is reused by the security
|
||||
# count-floor check and the smoke run below, avoiding a second compile of
|
||||
# the same e2e_test target on cold runners (backlog#1645).
|
||||
- name: Archive e2e smoke test binaries
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||
# profile.e2e-smoke default-filter in .config/nextest.toml is the single
|
||||
@@ -681,7 +685,30 @@ jobs:
|
||||
# adding new e2e jobs here. Each test spawns its own rustfs server on a
|
||||
# random port and reuses the downloaded debug binary above.
|
||||
- name: Run e2e smoke suite
|
||||
run: cargo nextest run --profile e2e-smoke -p e2e_test
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
run: |
|
||||
cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-smoke-diagnostics-${{ github.run_number }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-smoke-junit-${{ github.run_number }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Install s3s-e2e test tool
|
||||
uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
|
||||
|
||||
@@ -50,6 +50,21 @@ pub const ENV_RUSTFS_BUILD_FEATURES: &str = "RUSTFS_BUILD_FEATURES";
|
||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
|
||||
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
||||
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
||||
Some(log_dir.join(format!("{temp_name}.log")))
|
||||
}
|
||||
|
||||
fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
let log_dir = std::env::var_os("RUSTFS_E2E_LOG_DIR")?;
|
||||
if stdfs::create_dir_all(&log_dir).is_err() {
|
||||
warn!(?log_dir, "failed to create configured E2E server log directory");
|
||||
return None;
|
||||
}
|
||||
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, provider_name);
|
||||
let mut config = Config::builder()
|
||||
@@ -361,6 +376,7 @@ impl RustFSTestEnvironment {
|
||||
pub async fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
|
||||
fs::create_dir_all(&temp_dir).await?;
|
||||
let capture_log_path = configured_capture_log_path(&temp_dir);
|
||||
|
||||
// Use a unique port for each test environment
|
||||
let port = Self::find_available_port().await?;
|
||||
@@ -374,7 +390,7 @@ impl RustFSTestEnvironment {
|
||||
access_key: DEFAULT_ACCESS_KEY.to_string(),
|
||||
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
||||
process: None,
|
||||
capture_log_path: None,
|
||||
capture_log_path,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -382,6 +398,7 @@ impl RustFSTestEnvironment {
|
||||
pub async fn with_address(address: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
|
||||
fs::create_dir_all(&temp_dir).await?;
|
||||
let capture_log_path = configured_capture_log_path(&temp_dir);
|
||||
|
||||
let url = format!("http://{address}");
|
||||
|
||||
@@ -392,7 +409,7 @@ impl RustFSTestEnvironment {
|
||||
access_key: DEFAULT_ACCESS_KEY.to_string(),
|
||||
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
||||
process: None,
|
||||
capture_log_path: None,
|
||||
capture_log_path,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1392,6 +1409,14 @@ mod tests {
|
||||
assert_eq!(normalize_rustfs_build_features(" , "), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capture_log_path_uses_temp_directory_basename() {
|
||||
assert_eq!(
|
||||
capture_log_path(Path::new("/tmp/e2e-logs"), "/tmp/rustfs_e2e_test_abc"),
|
||||
Some(PathBuf::from("/tmp/e2e-logs/rustfs_e2e_test_abc.log"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_feature_enables_any_required_feature() {
|
||||
assert!(rustfs_build_feature_enabled(Some("full"), "sftp"));
|
||||
|
||||
@@ -30,8 +30,9 @@
|
||||
# Keep this regex in sync with the module names in the e2e-smoke filter.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/check_security_smoke_count.sh # count check (alias of check)
|
||||
# scripts/check_security_smoke_count.sh check # count check only
|
||||
# scripts/check_security_smoke_count.sh # count check (alias of check)
|
||||
# scripts/check_security_smoke_count.sh check # count check only
|
||||
# scripts/check_security_smoke_count.sh check listing.json # reuse a nextest listing
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -51,6 +52,17 @@ case "$mode" in
|
||||
;;
|
||||
esac
|
||||
|
||||
if (( $# > 2 )); then
|
||||
echo "usage: $0 [check] [nextest-listing.json]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
listing_file="${2:-}"
|
||||
if [[ -n "$listing_file" && ! -f "$listing_file" ]]; then
|
||||
echo "error: nextest listing does not exist: $listing_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
floor="$(grep -Ev '^[[:space:]]*(#|$)' "$FLOOR_FILE" | head -n1 | tr -d '[:space:]')"
|
||||
if ! [[ "$floor" =~ ^[0-9]+$ ]]; then
|
||||
echo "error: $FLOOR_FILE does not contain a numeric floor (got: '$floor')" >&2
|
||||
@@ -59,13 +71,20 @@ fi
|
||||
|
||||
# List via the e2e-smoke PROFILE so the default-filter is applied: only the
|
||||
# tests the PR smoke suite would actually run appear in the output. Count the
|
||||
# ones whose test name starts with a security module prefix. The structured JSON
|
||||
# listing is used (not the human format) for the same reason infra-12 does: the
|
||||
# human format's indentation varies across nextest versions; a JSON schema change
|
||||
# makes jq fail loudly rather than silently collapsing the count to zero.
|
||||
count="$(cargo nextest list --profile e2e-smoke -p e2e_test --message-format json \
|
||||
| jq --arg re "$SECURITY_SMOKE_REGEX" \
|
||||
'[."rust-suites"[].testcases | to_entries[] | select(.key | test($re))] | length')"
|
||||
# ones whose test name starts with a security module prefix. CI passes the
|
||||
# listing produced from the smoke archive so this guard does not rebuild the
|
||||
# e2e test graph immediately before the smoke run. The structured JSON listing
|
||||
# is used (not the human format) because a schema change makes jq fail loudly
|
||||
# rather than silently collapsing the count to zero.
|
||||
if [[ -n "$listing_file" ]]; then
|
||||
count="$(jq --arg re "$SECURITY_SMOKE_REGEX" \
|
||||
'[."rust-suites"[].testcases | to_entries[] | select(.value["filter-match"].status == "matches") | select(.key | test($re))] | length' \
|
||||
"$listing_file")"
|
||||
else
|
||||
count="$(cargo nextest list --profile e2e-smoke -p e2e_test --message-format json \
|
||||
| jq --arg re "$SECURITY_SMOKE_REGEX" \
|
||||
'[."rust-suites"[].testcases | to_entries[] | select(.value["filter-match"].status == "matches") | select(.key | test($re))] | length')"
|
||||
fi
|
||||
if ! [[ "$count" =~ ^[0-9]+$ ]]; then
|
||||
echo "error: could not parse nextest JSON listing (got count: '$count')" >&2
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user