mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-26 21:47:20 +00:00
rcourtman
b027bb0e8e
Since 15:00 UTC on 5 October every paired benchmark run that was not superseded has hit its 25-minute step timeout. Each step runs ten base and ten candidate samples, so per-sample growth is multiplied by twenty. BenchmarkRegistryProjectionSnapshot (#2518) rebuilt a registry behind StopTimer on every cold iteration. Go sizes b.N from timed work only, so the empty-registry cases ran hundreds of thousands of untimed ingests and took 16 s of a 100 ms benchtime. With #2486's new owner-index and metadata benchmarks, internal/unifiedresources went from 8 s to 38 s per sample. Cold now invalidates the registry's derived state instead. The refresh never reads its own output, so this repeats a fresh ingest's first read with the same allocations, and the family drops from 22 s to about 5 s per sample. Paired runs also measured every package even when both trees compiled identical code, as on frontend-only changes. They now sample only packages whose in-module dependency closure the change reaches. Module files, dirty or non-Git trees and an unreadable graph still select everything. A module or shared-model change still samples nearly every package, about 24 minutes with warm-up. The step therefore gets a measured 30 minutes, and the script stops at a 28-minute projected budget with per-package costs instead of meeting a bare timeout.
256 lines
10 KiB
Bash
Executable File
256 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Contract tests for paired CI benchmark collection and verdict quality.
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
WORK_DIR="$(mktemp -d)"
|
|
trap 'rm -rf "${WORK_DIR}"' EXIT
|
|
|
|
mkdir -p "${WORK_DIR}/bin" "${WORK_DIR}/candidate" "${WORK_DIR}/baseline"
|
|
cat > "${WORK_DIR}/bin/fixture.test" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
[[ "$1" == '-test.bench=.' ]] || exit 91
|
|
[[ "${FAKE_BINARY_FAIL:-0}" == 0 ]] || exit 23
|
|
echo 'BenchmarkExample-4 1 100 ns/op 0 B/op 0 allocs/op'
|
|
EOF
|
|
chmod +x "${WORK_DIR}/bin/fixture.test"
|
|
export FAKE_BINARY="${WORK_DIR}/bin/fixture.test"
|
|
cat > "${WORK_DIR}/bin/go" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
if [[ "$*" == version ]]; then
|
|
echo 'go version go1.26.7 linux/amd64'
|
|
exit 0
|
|
fi
|
|
# Every package's closure is itself plus internal/shared, except pkg/auth.
|
|
if [[ "$1" == list ]]; then
|
|
[[ "${FAKE_GO_LIST_FAIL:-0}" == 0 ]] || exit 1
|
|
if [[ "$2" == -m ]]; then
|
|
echo "$PWD"
|
|
exit 0
|
|
fi
|
|
package="${!#}"
|
|
package="${package#./}"
|
|
package="${package%/}"
|
|
printf '%s/%s\n\n' "$PWD" "${package}"
|
|
[[ "${package}" == pkg/auth ]] || echo "$PWD/internal/shared"
|
|
exit 0
|
|
fi
|
|
printf '%s\t%s\n' "$PWD" "$*" >> "${FAKE_GO_LOG}"
|
|
args=("$@")
|
|
for ((i=0; i<${#args[@]}; i++)); do
|
|
if [[ "${args[i]}" == -exec ]]; then
|
|
"${args[i+1]}" "${FAKE_BINARY}" '-test.bench=.' || exit
|
|
printf 'ok \texample.test/bench\t1.500s\n'
|
|
exit 0
|
|
fi
|
|
done
|
|
cat <<'RESULT'
|
|
goos: linux
|
|
goarch: amd64
|
|
pkg: example.test/bench
|
|
cpu: test
|
|
BenchmarkExample-4 1 100 ns/op 0 B/op 0 allocs/op
|
|
PASS
|
|
ok example.test/bench 0.001s
|
|
RESULT
|
|
EOF
|
|
chmod +x "${WORK_DIR}/bin/go"
|
|
|
|
(
|
|
cd "${WORK_DIR}/candidate"
|
|
PATH="${WORK_DIR}/bin:${PATH}" \
|
|
FAKE_GO_LOG="${WORK_DIR}/go.log" \
|
|
PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
|
|
PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" \
|
|
PULSE_BENCH_SAMPLE_COUNT=2 \
|
|
bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
|
|
)
|
|
|
|
[[ "$(grep -c '^BenchmarkExample' "${WORK_DIR}/candidate/bench-baseline.txt")" == 2 ]]
|
|
[[ "$(grep -c '^BenchmarkExample' "${WORK_DIR}/candidate/bench-results.txt")" == 2 ]]
|
|
|
|
# Warm-up is baseline then candidate. Measured rounds must alternate which tree
|
|
# goes first, avoiding a permanent order advantage.
|
|
mapfile -t calls < "${WORK_DIR}/go.log"
|
|
[[ "${#calls[@]}" == 6 ]]
|
|
[[ "${calls[2]}" == "${WORK_DIR}/baseline"$'\t'* ]]
|
|
[[ "${calls[3]}" == "${WORK_DIR}/candidate"$'\t'* ]]
|
|
[[ "${calls[4]}" == "${WORK_DIR}/candidate"$'\t'* ]]
|
|
[[ "${calls[5]}" == "${WORK_DIR}/baseline"$'\t'* ]]
|
|
|
|
metadata="${WORK_DIR}/candidate/bench-metadata.txt"
|
|
grep -qFx 'samples=2' "${metadata}"
|
|
grep -qFx 'selection=all (trees are not both clean Git checkouts)' "${metadata}"
|
|
grep -qFx 'packages=./pkg/metrics/ ./pkg/auth/ ./internal/api/ ./internal/monitoring/ ./internal/unifiedresources/ ./internal/dockeragent/ ./cmd/pulse-agent/ ./internal/hostagent/ ./internal/hostmetrics/' "${metadata}"
|
|
grep -qFx 'benchtime=100ms' "${metadata}"
|
|
grep -qFx 'candidate.commit=unavailable' "${metadata}"
|
|
grep -qFx 'baseline.go=go version go1.26.7 linux/amd64' "${metadata}"
|
|
[[ "$(grep -c '^sample=' "${metadata}")" == 4 ]]
|
|
! grep -qF "${WORK_DIR}" "${metadata}"
|
|
digest="$(sha256sum "${FAKE_BINARY}")"
|
|
digest="${digest%% *}"
|
|
[[ "$(grep -c '^binary=' "${metadata}")" == 4 ]]
|
|
for tuple in baseline,1 candidate,1 candidate,2 baseline,2; do
|
|
grep -qFx "binary=${tuple},fixture.test,${digest}" "${metadata}"
|
|
done
|
|
|
|
# Real Git roots retain exact identities and measure only the packages whose
|
|
# dependency closure the candidate changes; a repeated run replaces metadata.
|
|
for tree in candidate baseline; do
|
|
git -C "${WORK_DIR}/${tree}" init -q
|
|
mkdir -p "${WORK_DIR}/${tree}/internal/shared" "${WORK_DIR}/${tree}/internal/api"
|
|
echo 'module example.test' > "${WORK_DIR}/${tree}/go.mod"
|
|
echo 'package shared' > "${WORK_DIR}/${tree}/internal/shared/shared.go"
|
|
echo 'package api' > "${WORK_DIR}/${tree}/internal/api/handler.go"
|
|
done
|
|
change() {
|
|
mkdir -p "$(dirname "${WORK_DIR}/candidate/$1")"
|
|
echo "// $2" >> "${WORK_DIR}/candidate/$1"
|
|
}
|
|
commit() {
|
|
git -C "${WORK_DIR}/$1" add go.mod internal frontend 2>/dev/null || git -C "${WORK_DIR}/$1" add go.mod internal
|
|
git -C "${WORK_DIR}/$1" -c user.name=Test -c user.email=test@example.invalid commit -qm fixture
|
|
}
|
|
run_paired() {
|
|
: > "${WORK_DIR}/go.log"
|
|
PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
|
|
PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
|
|
PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=1 \
|
|
bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" "$@" >/dev/null
|
|
}
|
|
commit baseline
|
|
|
|
# Frontend files and another package's tests reach no benchmark binary.
|
|
change frontend/app.ts frontend
|
|
change internal/shared/shared_test.go shared-test
|
|
commit candidate
|
|
run_paired
|
|
grep -qFx 'selection=affected' "${metadata}"
|
|
grep -qFx 'packages=' "${metadata}"
|
|
grep -qFx "candidate.commit=$(git -C "${WORK_DIR}/candidate" rev-parse HEAD)" "${metadata}"
|
|
grep -qFx "baseline.tree=$(git -C "${WORK_DIR}/baseline" rev-parse 'HEAD^{tree}')" "${metadata}"
|
|
[[ "$(grep -c '^sample=' "${metadata}" || true)" == 0 ]]
|
|
[[ ! -s "${WORK_DIR}/candidate/bench-results.txt" ]]
|
|
[[ ! -s "${WORK_DIR}/candidate/bench-baseline.txt" ]]
|
|
[[ ! -s "${WORK_DIR}/go.log" ]]
|
|
|
|
# A benchmark package's own test file selects that package alone.
|
|
change internal/api/handler_test.go api-test
|
|
commit candidate
|
|
run_paired
|
|
grep -qFx 'packages=./internal/api/' "${metadata}"
|
|
[[ "$(grep -c '^sample=' "${metadata}")" == 2 ]]
|
|
[[ "$(grep -c '^BenchmarkExample' "${WORK_DIR}/candidate/bench-results.txt")" == 1 ]]
|
|
mapfile -t calls < "${WORK_DIR}/go.log"
|
|
[[ "${#calls[@]}" == 4 ]]
|
|
for call in "${calls[@]}"; do
|
|
[[ "${call}" == *' ./internal/api/' && "${call}" != *pkg/metrics* ]]
|
|
done
|
|
|
|
# A shared non-test file selects every package that imports its directory.
|
|
change internal/shared/shared.go shared
|
|
commit candidate
|
|
run_paired
|
|
grep -qFx 'selection=affected' "${metadata}"
|
|
grep -qFx 'packages=./pkg/metrics/ ./internal/api/ ./internal/monitoring/ ./internal/unifiedresources/ ./internal/dockeragent/ ./cmd/pulse-agent/ ./internal/hostagent/ ./internal/hostmetrics/' "${metadata}"
|
|
|
|
# An unreadable package graph measures everything rather than guessing.
|
|
FAKE_GO_LIST_FAIL=1 run_paired
|
|
grep -qFx 'selection=all (package graph unreadable)' "${metadata}"
|
|
grep -qF 'packages=./pkg/metrics/ ./pkg/auth/' "${metadata}"
|
|
|
|
# Uncommitted edits are measured but invisible to the committed trees.
|
|
echo '// local edit' >> "${WORK_DIR}/baseline/internal/api/handler.go"
|
|
run_paired
|
|
grep -qFx 'selection=all (trees are not both clean Git checkouts)' "${metadata}"
|
|
git -C "${WORK_DIR}/baseline" checkout -q -- internal/api/handler.go
|
|
|
|
# Module changes can move any package.
|
|
change go.mod module
|
|
commit candidate
|
|
run_paired
|
|
grep -qFx 'selection=all (module files changed)' "${metadata}"
|
|
grep -qF 'packages=./pkg/metrics/ ./pkg/auth/' "${metadata}"
|
|
|
|
# A nested archive has no identity of its own; never attribute its parent's SHA.
|
|
mkdir -p "${WORK_DIR}/candidate/archive"
|
|
PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
|
|
PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate/archive" \
|
|
PULSE_BENCH_BASELINE_DIR='' PULSE_BENCH_SAMPLE_COUNT=1 \
|
|
bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
|
|
grep -qFx 'candidate.commit=unavailable' "${WORK_DIR}/candidate/archive/bench-metadata.txt"
|
|
! grep -q '^baseline\.' "${WORK_DIR}/candidate/archive/bench-metadata.txt"
|
|
grep -qFx 'selection=all (no baseline)' "${WORK_DIR}/candidate/archive/bench-metadata.txt"
|
|
|
|
grep -qFx "binary=candidate,unpaired,fixture.test,${digest}" \
|
|
"${WORK_DIR}/candidate/archive/bench-metadata.txt"
|
|
|
|
# The wrapper must not turn a failed executable into successful evidence.
|
|
set +e
|
|
PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
|
|
FAKE_BINARY_FAIL=1 PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
|
|
PULSE_BENCH_BASELINE_DIR='' PULSE_BENCH_SAMPLE_COUNT=1 \
|
|
bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
|
|
status=$?
|
|
set -e
|
|
[[ "${status}" == 23 ]]
|
|
grep -qFx "binary=candidate,unpaired,fixture.test,${digest}" "${metadata}"
|
|
|
|
# A run projected to reach its budget stops after the round that shows it,
|
|
# naming each package's mean cost; one inside its budget completes.
|
|
set +e
|
|
output="$(PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
|
|
PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
|
|
PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=3 \
|
|
PULSE_BENCH_BUDGET_SECONDS=0 \
|
|
bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" 2>&1 >/dev/null)"
|
|
status=$?
|
|
set -e
|
|
[[ "${status}" == 1 ]]
|
|
grep -qF 'Benchmark budget exceeded: 1 of 3 paired rounds' <<<"${output}"
|
|
grep -qE '^ +1\.5 example\.test/bench$' <<<"${output}"
|
|
[[ "$(grep -c '^sample=' "${metadata}")" == 2 ]]
|
|
PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
|
|
PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
|
|
PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=3 \
|
|
PULSE_BENCH_BUDGET_SECONDS=3600 \
|
|
bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
|
|
[[ "$(grep -c '^sample=' "${metadata}")" == 6 ]]
|
|
set +e
|
|
PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" PULSE_BENCH_BUDGET_SECONDS=soon \
|
|
bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null 2>&1
|
|
status=$?
|
|
set -e
|
|
[[ "${status}" == 2 ]]
|
|
|
|
cat > "${WORK_DIR}/adequate.txt" <<'EOF'
|
|
Example-4 100.0n ± 1% 111.0n ± 1% +11.00% (p=0.001 n=10)
|
|
EOF
|
|
set +e
|
|
output="$(bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/adequate.txt" 2>&1)"
|
|
status=$?
|
|
set -e
|
|
if [[ "${status}" == 0 ]]; then
|
|
echo "adequately sampled regression was not rejected" >&2
|
|
exit 1
|
|
fi
|
|
grep -qF "BENCHMARK REGRESSION DETECTED" <<<"${output}"
|
|
|
|
cat > "${WORK_DIR}/undersampled.txt" <<'EOF'
|
|
Example-4 100.0n ± ∞ ¹ 111.0n ± ∞ ¹ +11.00% (p=0.008 n=5)
|
|
EOF
|
|
set +e
|
|
output="$(bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/undersampled.txt" 2>&1)"
|
|
status=$?
|
|
set -e
|
|
[[ "${status}" != 0 ]]
|
|
grep -qF "BENCHMARK EVIDENCE INSUFFICIENT" <<<"${output}"
|
|
|
|
cat > "${WORK_DIR}/clean.txt" <<'EOF'
|
|
Example-4 100.0n ± 1% 105.0n ± 1% +5.00% (p=0.001 n=10)
|
|
EOF
|
|
bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/clean.txt" >/dev/null
|
|
|
|
echo "all CI benchmark tests passed"
|