diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index cdb9de7e0..5563383e1 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -111,9 +111,13 @@ jobs: - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh + - name: Check CI paths stay in sync run: ./scripts/check_ci_paths_sync.sh + - name: Check io_uring lane --lib precondition + run: ./scripts/check_uring_lane_lib_only.sh + test-and-lint: name: Test and Lint runs-on: ubuntu-latest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 085ed83a4..45f462b0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,9 +144,13 @@ jobs: - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh + - name: Check CI paths stay in sync run: ./scripts/check_ci_paths_sync.sh + - name: Check io_uring lane --lib precondition + run: ./scripts/check_uring_lane_lib_only.sh + test-and-lint: name: Test and Lint if: github.event_name != 'pull_request' || github.event.action != 'closed' @@ -197,8 +201,14 @@ jobs: # Clippy runs before the test pass: lint failures are the most common # CI-only breakage and should surface in minutes, not after 20+ minutes # of tests. + # Sampled too: clippy is the natural control arm for any CARGO_BUILD_JOBS + # experiment, since --all-targets is check-only for workspace members and + # never links the ~100 test binaries the limit exists to throttle. - name: Run clippy lints - run: cargo clippy --all-targets -- -D warnings + run: | + ./scripts/ci/resource_sampler.sh start clippy + trap './scripts/ci/resource_sampler.sh stop' EXIT + cargo clippy --all-targets -- -D warnings - name: Run nextest tests env: @@ -207,33 +217,8 @@ jobs: CARGO_BUILD_JOBS: "2" run: | mkdir -p artifacts/test-and-lint - # Evidence sampler for issue #5394: the post-mortem pgrep below runs - # only after `timeout` has already TERM'd the whole cargo process - # group, so it cannot name a wedged process. Sample system and - # process state every 60s instead; the last samples before the - # timeout show what was stuck (rustc, linker, build script, memory - # pressure, ...). The log rides along in the existing artifact. - ( - while true; do - { - echo "=== $(date --utc --iso-8601=seconds)" - echo "--- load"; cat /proc/loadavg - echo "--- psi"; grep -H . /proc/pressure/* 2>/dev/null || true - echo "--- mem"; free -m - echo "--- disk"; df -h / /home/runner 2>/dev/null || df -h / - echo "--- top-rss" - ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss | head -15 - echo "--- build/test processes" - ps -eo pid,ppid,stat,etime,rss,pcpu,args | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true - echo "--- d-state (uninterruptible IO)" - ps -eo pid,stat,etime,args | awk 'NR > 1 && $2 ~ /D/' || true - echo - } >> artifacts/test-and-lint/sampler.log 2>&1 || true - sleep 60 - done - ) & - sampler_pid=$! - trap 'kill "${sampler_pid}" 2>/dev/null || true' EXIT + ./scripts/ci/resource_sampler.sh start nextest + trap './scripts/ci/resource_sampler.sh stop' EXIT set +e NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \ cargo nextest run --profile ci --all --exclude e2e_test \ @@ -592,7 +577,17 @@ jobs: RUSTFS_IO_URING_READ_ENABLE: "true" RUSTFS_URING_TESTS_MUST_RUN: "1" TMPDIR: /mnt/rustfs-odirect - run: cargo test -p rustfs-ecstore uring_ -- --test-threads=1 --nocapture + # --lib narrows what gets compiled, not what gets run: every selected + # test lives in the lib target. The 7 integration binaries under + # crates/ecstore/tests/ each reported "running 0 tests" here, so they + # were compiled and linked for nothing. + # + # The `uring_` filter must stay exactly as it is. libtest matches on + # substring, so it also selects names containing `during_` — 6 of the 18 + # selected tests are such incidental matches. Narrowing the filter to + # `io_uring` would silently drop them, which is a coverage change. + # scripts/check_uring_lane_lib_only.sh guards the --lib precondition. + run: cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture e2e-tests: name: End-to-End Tests diff --git a/scripts/check_uring_lane_lib_only.sh b/scripts/check_uring_lane_lib_only.sh new file mode 100755 index 000000000..a862d996b --- /dev/null +++ b/scripts/check_uring_lane_lib_only.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# The io_uring CI lane runs `cargo test -p rustfs-ecstore --lib uring_`. +# +# `--lib` is there to avoid compiling and linking the 7 integration binaries +# under crates/ecstore/tests/, which selected zero tests for this filter. That is +# only safe while it stays true: add a matching test under tests/ and `--lib` +# would skip it silently, with CI staying green — the failure mode is invisible, +# so it is asserted here instead. +# +# The pattern must use the same `uring_` substring semantics as libtest, which +# also matches names containing `during_`. Narrowing it to `io_uring` would make +# this guard disagree with the filter it is protecting. +# +# Usage: scripts/check_uring_lane_lib_only.sh +set -euo pipefail + +cd "$(dirname "$0")/.." + +TESTS_DIR="crates/ecstore/tests" + +if [ ! -d "$TESTS_DIR" ]; then + echo "OK: $TESTS_DIR does not exist; nothing for --lib to miss" + exit 0 +fi + +hits="$(grep -rEn '^[[:space:]]*(pub[[:space:]]+)?(async[[:space:]]+)?fn[[:space:]]+[A-Za-z0-9_]*uring_' \ + "$TESTS_DIR" || true)" + +if [ -n "$hits" ]; then + echo "ERROR: test functions matching the 'uring_' substring found under $TESTS_DIR:" >&2 + printf '%s\n' "$hits" | sed 's/^/ /' >&2 + echo "" >&2 + echo "The io_uring lane in .github/workflows/ci.yml uses --lib, so these would" >&2 + echo "never run and CI would stay green. Pick one:" >&2 + echo " - move them into a #[cfg(test)] mod under crates/ecstore/src, or" >&2 + echo " - drop --lib from that step (and pay the link cost for all 7 binaries), or" >&2 + echo " - add an explicit --test for the binary and update this script." >&2 + exit 1 +fi + +echo "OK: no 'uring_'-matching test functions under $TESTS_DIR; --lib is safe" diff --git a/scripts/ci/resource_sampler.sh b/scripts/ci/resource_sampler.sh new file mode 100755 index 000000000..4c3148848 --- /dev/null +++ b/scripts/ci/resource_sampler.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Background resource sampler for the Test and Lint lane (issue #5394). +# +# The post-mortem `pgrep` in ci.yml runs only after `timeout` has already TERM'd +# the whole cargo process group, so it can never name a wedged process. This +# samples every 60s instead; the last samples before a timeout show what was +# stuck — rustc, the linker, a build script, memory pressure. +# +# WHY THE CGROUP READINGS MATTER +# +# The runners are ARC (Actions Runner Controller) pods, so /proc/loadavg, +# /proc/pressure/*, `free` and `df` are all *node*-level: they include every +# other runner pod scheduled on the same Kubernetes node. A sample showing high +# I/O pressure says nothing about whether *this* job caused it. Measured +# evidence: one sample reported loadavg 6.67 with 832 threads node-wide while +# `ps` inside the pod showed about 10 processes. +# +# Only the /sys/fs/cgroup/* values and `ps` are attributable to this job, so +# those are what any CARGO_BUILD_JOBS experiment must be judged on. The +# node-level readings are kept — co-tenancy is itself a real cause of stalls, +# and a 9m57s `git checkout` was traced to it — but labelled NODE-LEVEL so +# nobody reads them as this pod's own load. +# +# Usage: +# scripts/ci/resource_sampler.sh start # e.g. clippy, nextest, doctest +# scripts/ci/resource_sampler.sh stop +# +# Safe to call `stop` when nothing is running, and safe to `start` a new phase +# while another is sampling — the previous one is stopped first. Never fails the +# calling step: this is diagnostics, not a gate. +set -uo pipefail + +# GNU date on the runners; the fallback keeps local smoke tests on macOS quiet. +now() { date --utc --iso-8601=seconds 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%S+00:00; } + +LOG_DIR="artifacts/test-and-lint" +LOG_FILE="${LOG_DIR}/sampler.log" +PID_FILE="${LOG_DIR}/.sampler.pid" +INTERVAL="${SAMPLER_INTERVAL_SECS:-60}" + +stop_sampler() { + [ -f "$PID_FILE" ] || return 0 + local pid + pid="$(cat "$PID_FILE" 2>/dev/null || true)" + if [ -n "${pid:-}" ]; then + kill "$pid" 2>/dev/null || true + fi + rm -f "$PID_FILE" +} + +sample_once() { + echo "=== $(now)" + + # Attributable to this pod: cgroup v2 accounting and our own process table. + echo "--- POD cpu/mem limits" + grep -H . /sys/fs/cgroup/cpu.max /sys/fs/cgroup/memory.max \ + /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current 2>/dev/null || true + echo "--- POD pressure (cgroup v2, this pod only)" + grep -H . /sys/fs/cgroup/cpu.pressure /sys/fs/cgroup/io.pressure \ + /sys/fs/cgroup/memory.pressure 2>/dev/null || true + echo "--- POD nproc"; nproc 2>/dev/null || true + + # NODE-LEVEL: shared with every other runner pod on this Kubernetes node. + # Useful for spotting co-tenancy, useless for attributing load to this job. + echo "--- NODE-LEVEL load (shared with co-tenant runners)"; cat /proc/loadavg 2>/dev/null || true + echo "--- NODE-LEVEL psi (shared with co-tenant runners)" + grep -H . /proc/pressure/* 2>/dev/null || true + echo "--- NODE-LEVEL mem (shared with co-tenant runners)"; free -m 2>/dev/null || true + echo "--- NODE-LEVEL disk (shared with co-tenant runners)" + df -h / /home/runner 2>/dev/null || df -h / 2>/dev/null || true + + echo "--- top-rss" + ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss 2>/dev/null | head -15 || true + echo "--- build/test processes" + ps -eo pid,ppid,stat,etime,rss,pcpu,args 2>/dev/null \ + | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true + echo "--- d-state (uninterruptible IO)" + ps -eo pid,stat,etime,args 2>/dev/null | awk 'NR > 1 && $2 ~ /D/' || true + echo +} + +case "${1:-}" in + start) + phase="${2:-unknown}" + mkdir -p "$LOG_DIR" + stop_sampler + { + echo "--- phase=${phase} started_at=$(now) runner=${RUNNER_NAME:-unknown}" + } >> "$LOG_FILE" 2>&1 || true + ( + while true; do + sample_once >> "$LOG_FILE" 2>&1 || true + sleep "$INTERVAL" + done + ) & + echo $! > "$PID_FILE" + ;; + stop) + stop_sampler + ;; + *) + echo "usage: $0 start | stop" >&2 + exit 2 + ;; +esac + +exit 0