From b027bb0e8e51b13faf8e2a2279001c51737dba74 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:39:28 +0100 Subject: [PATCH] Keep paired PR benchmarks inside their time budget Since 15:00 UTC on 5 October every paired benchmark run that was not superseded has hit its 25-minute step timeout. Each step runs ten base and ten candidate samples, so per-sample growth is multiplied by twenty. BenchmarkRegistryProjectionSnapshot (#2518) rebuilt a registry behind StopTimer on every cold iteration. Go sizes b.N from timed work only, so the empty-registry cases ran hundreds of thousands of untimed ingests and took 16 s of a 100 ms benchtime. With #2486's new owner-index and metadata benchmarks, internal/unifiedresources went from 8 s to 38 s per sample. Cold now invalidates the registry's derived state instead. The refresh never reads its own output, so this repeats a fresh ingest's first read with the same allocations, and the family drops from 22 s to about 5 s per sample. Paired runs also measured every package even when both trees compiled identical code, as on frontend-only changes. They now sample only packages whose in-module dependency closure the change reaches. Module files, dirty or non-Git trees and an unreadable graph still select everything. A module or shared-model change still samples nearly every package, about 24 minutes with warm-up. The step therefore gets a measured 30 minutes, and the script stops at a 28-minute projected budget with per-package costs instead of meeting a bare timeout. --- .github/workflows/build-and-test.yml | 8 +- .../subsystems/deployment-installability.md | 10 ++ .../registry_projection_test.go | 15 +- scripts/BENCHMARK_EVIDENCE.md | 24 +++ .../installtests/build_release_assets_test.go | 32 ++++ scripts/run-ci-benchmarks.sh | 143 ++++++++++++++++-- scripts/tests/test-ci-benchmarks.sh | 126 ++++++++++++++- 7 files changed, 335 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7a0d49b3a2..0699a5536a 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -546,12 +546,18 @@ jobs: PULSE_BENCH_SAMPLE_COUNT: "10" run: bash scripts/run-ci-benchmarks.sh + # Only packages whose dependency closure the pull request changes are + # sampled. A module or shared-model change still samples nearly every + # package: about 24 minutes, warm-up included, as of October 2026. The + # script's budget stops the run with per-package costs before the step + # timeout can kill it without them. - name: Paired base and candidate benchmarks (pull requests) if: github.event_name == 'pull_request' - timeout-minutes: 25 + timeout-minutes: 30 env: PULSE_BENCH_BASELINE_DIR: ${{ github.workspace }}/benchmark-base PULSE_BENCH_SAMPLE_COUNT: "10" + PULSE_BENCH_BUDGET_SECONDS: "1680" run: bash scripts/run-ci-benchmarks.sh - name: Install benchstat diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 703c05d3d1..68b7e24d41 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -330,6 +330,16 @@ wiring is checked by `TestBenchmarkQualificationRetainsProvenance` in `scripts/installtests/build_release_assets_test.go`; executed collection cases remain in `scripts/tests/test-ci-benchmarks.sh`. +Paired pull-request runs sample only the packages whose in-module dependency +closure the change reaches, recorded as `selection=` and `packages=` in the +metadata. Module files, unclean or non-Git trees and an unreadable package +graph select every package; unpaired branch runs always sample every package. +An empty selection leaves nothing to compare and is not a passed comparison of +skipped packages. The paired step's wall-clock budget sits below its timeout so +an outgrown suite stops with each package's mean cost per sample, not a bare +timeout; `TestPairedBenchmarksStopInsideTheirBudget` checks that ordering and +the selection and budget cases run in `scripts/tests/test-ci-benchmarks.sh`. + ### Public Helm exact-package receipt The post-activation public Pages verification in `.github/workflows/helm-pages.yml` diff --git a/internal/unifiedresources/registry_projection_test.go b/internal/unifiedresources/registry_projection_test.go index ca9dda8383..3109d25296 100644 --- a/internal/unifiedresources/registry_projection_test.go +++ b/internal/unifiedresources/registry_projection_test.go @@ -66,10 +66,17 @@ func BenchmarkRegistryProjectionSnapshot(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { if mode == "cold" { - b.StopTimer() - rr = NewRegistry(nil) - rr.IngestSnapshot(state) - b.StartTimer() + // Return to the state IngestSnapshot leaves: canonical + // metadata, typed views and the source-target index all + // pending. The refresh never reads its own prior output, + // so this repeats a fresh ingest's first read exactly. + // Rebuilding the registry behind StopTimer instead let a + // nanosecond empty-registry read push b.N into the + // hundreds of thousands, each paying an untimed ingest + // and two stop-the-world timer reads. + rr.mu.Lock() + rr.invalidateSourceTargetsLocked() + rr.mu.Unlock() } if mode == "dirty" { rr.mu.Lock() diff --git a/scripts/BENCHMARK_EVIDENCE.md b/scripts/BENCHMARK_EVIDENCE.md index 8f2d69d1ea..e90973e1b5 100644 --- a/scripts/BENCHMARK_EVIDENCE.md +++ b/scripts/BENCHMARK_EVIDENCE.md @@ -38,3 +38,27 @@ These hashes do not retain binaries or establish equivalent instructions, linked data, host state or performance. They make later reconstructions checkable; a hash mismatch means they are not the original measured bytes. Historical failures without hashes remain unresolved evidence. + +Paired runs sample only packages whose in-module dependency closure (`go list +-deps -test` in the candidate tree) holds a file whose committed content +differs between the two HEADs. A changed `_test.go` file selects only its own +package, since no other test binary compiles it; any other file selects every +package whose closure holds its directory or a parent of it, which covers +embeds and testdata. Module or workspace files, trees that are not both clean +Git roots, or an unreadable package graph select every package. Metadata `selection=` names the rule applied and +`packages=` the packages sampled; an empty list leaves empty timing files and +nothing to compare. A benchmark reading files outside that closure at run time +escapes selection. Unpaired runs always sample every package. + +With `PULSE_BENCH_BUDGET_SECONDS` set, a paired run projects its total, +warm-up included, after each round and exits 1 once the projection reaches the +budget, printing each package's mean seconds per sample. CI keeps the budget +below the step timeout so an outgrown suite fails with that attribution, not a +bare timeout. Partial timing files are uploaded but are not a comparison. + +Keep untimed work per iteration in proportion to timed work. Setup between +`b.StopTimer` and `b.StartTimer` is excluded from the estimate Go uses to pick +`b.N`, so a nanosecond operation is repeated hundreds of thousands of times +with all of that setup. `BenchmarkRegistryProjectionSnapshot`'s empty cold cases +took 16 s per sample at a 100 ms benchtime that way and pushed paired runs past +their timeout. diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index c7ff12b598..177dac13b8 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -3874,6 +3874,38 @@ func TestBenchmarkQualificationRetainsProvenance(t *testing.T) { } } +// A paired run names each package's cost only when its own budget stops it +// before the step timeout does, and the step must end inside its job. +func TestPairedBenchmarksStopInsideTheirBudget(t *testing.T) { + content, err := os.ReadFile(repoFile(".github", "workflows", "build-and-test.yml")) + if err != nil { + t.Fatal(err) + } + job := workflowJobBlock(t, string(content), "benchmarks") + step := workflowStepBlock(t, job, "Paired base and candidate benchmarks (pull requests)") + value := func(block, pattern, what string) int { + t.Helper() + match := regexp.MustCompile(pattern).FindStringSubmatch(block) + if match == nil { + t.Fatalf("benchmark workflow missing %s", what) + } + n, err := strconv.Atoi(match[1]) + if err != nil { + t.Fatal(err) + } + return n + } + jobMinutes := value(job, `(?m)^ timeout-minutes: (\d+)$`, "job timeout") + stepMinutes := value(step, `(?m)^ timeout-minutes: (\d+)$`, "paired step timeout") + budget := value(step, `(?m)^ PULSE_BENCH_BUDGET_SECONDS: "(\d+)"$`, "paired budget") + if budget >= stepMinutes*60 { + t.Fatalf("paired budget %ds must stop before the %d-minute step timeout", budget, stepMinutes) + } + if stepMinutes >= jobMinutes { + t.Fatalf("paired step timeout %dm must end inside the %dm job", stepMinutes, jobMinutes) + } +} + func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) { content, err := os.ReadFile(repoFile(".github", "workflows", "build-and-test.yml")) if err != nil { diff --git a/scripts/run-ci-benchmarks.sh b/scripts/run-ci-benchmarks.sh index 12e3fe0fac..7a20e85128 100755 --- a/scripts/run-ci-benchmarks.sh +++ b/scripts/run-ci-benchmarks.sh @@ -9,11 +9,19 @@ CURRENT_DIR="${PULSE_BENCH_CURRENT_DIR:-$(pwd)}" BASELINE_DIR="${PULSE_BENCH_BASELINE_DIR:-}" SAMPLE_COUNT="${PULSE_BENCH_SAMPLE_COUNT:-10}" BENCHTIME="${PULSE_BENCH_BENCHTIME:-100ms}" +# Wall-clock seconds a paired run may take, warm-up included; unset means no +# limit. Checked after every round, so a suite that has outgrown its budget +# stops early and names each package's cost instead of meeting a step timeout. +BUDGET_SECONDS="${PULSE_BENCH_BUDGET_SECONDS:-}" if ! [[ "${SAMPLE_COUNT}" =~ ^[1-9][0-9]*$ ]]; then echo "PULSE_BENCH_SAMPLE_COUNT must be a positive integer." >&2 exit 2 fi +if [[ -n "${BUDGET_SECONDS}" && ! "${BUDGET_SECONDS}" =~ ^[0-9]+$ ]]; then + echo "PULSE_BENCH_BUDGET_SECONDS must be a whole number of seconds." >&2 + exit 2 +fi if [[ ! -d "${CURRENT_DIR}" ]]; then echo "Benchmark candidate tree not found: ${CURRENT_DIR}" >&2 exit 2 @@ -35,14 +43,105 @@ PACKAGES=( ./internal/hostmetrics/ ) +work_dir="$(mktemp -d)" +trap 'rm -rf "${work_dir}"' EXIT + +# An archived tree nested inside another checkout must not inherit its SHA. +is_git_root() { + local prefix + prefix="$(git -C "$1" rev-parse --show-prefix 2>/dev/null)" && [[ -z "${prefix}" ]] +} + +# Paths whose committed content differs between the two trees. Uncommitted +# edits to tracked files would be measured but not seen, so they fail. +changed_paths() { + local label tree + for label in baseline candidate; do + tree="${CURRENT_DIR}" + [[ "${label}" != baseline ]] || tree="${BASELINE_DIR}" + is_git_root "${tree}" || return 1 + [[ -z "$(git -C "${tree}" status --porcelain --untracked-files=no)" ]] || return 1 + git -C "${tree}" ls-tree -r --full-tree HEAD | LC_ALL=C sort > "${work_dir}/${label}.files" || + return 1 + done + LC_ALL=C comm -3 "${work_dir}/baseline.files" "${work_dir}/candidate.files" | + awk -F '\t' '{ print $NF }' | LC_ALL=C sort -u +} + +# Exit 0 when a changed path reaches the package's benchmarks, 1 when none +# does, and 2 when the package graph cannot be read. A test file compiles only +# into its own package's test binary; any other file reaches every package +# that imports its directory, including through embeds and testdata below it. +package_affected() { + local package="$1" changed="$2" closure="${work_dir}/closure" root + root="$(cd "${CURRENT_DIR}" && go list -m -f '{{.Dir}}')" || return 2 + (cd "${CURRENT_DIR}" && go list -e -deps -test \ + -f '{{with .Module}}{{if .Main}}{{$.Dir}}{{end}}{{end}}' "${package}") > "${closure}" || + return 2 + [[ -s "${closure}" ]] || return 2 + package="${package#./}" + awk -v root="${root}" -v own="${package%/}" ' + FILENAME == ARGV[1] { + if ($0 == root) dirs["."] + else if (index($0, root "/") == 1) dirs[substr($0, length(root) + 2)] + next + } + /_test\.go$/ { + dir = $0 + if (!sub(/\/[^\/]*$/, "", dir)) dir = "." + if (dir == own) { hit = 1; exit } + next + } + { + if ("." in dirs) { hit = 1; exit } + dir = $0 + while (sub(/\/[^\/]*$/, "", dir)) if (dir in dirs) { hit = 1; exit } + } + END { exit hit ? 0 : 1 } + ' "${closure}" "${changed}" +} + +# A paired run measures only packages whose in-module dependency closure the +# candidate changes. Elsewhere both trees compile identical code, so samples +# add runner time and noise but no evidence. Module files, or trees or a +# package graph that cannot be read, select every package. +SELECTED=("${PACKAGES[@]}") +selection="all (no baseline)" +select_packages() { + local changed="${work_dir}/changed-paths" package status picked=() + if ! changed_paths > "${changed}"; then + selection="all (trees are not both clean Git checkouts)" + return + fi + if grep -qxE 'go\.(mod|sum|work|work\.sum)' "${changed}"; then + selection="all (module files changed)" + return + fi + for package in "${PACKAGES[@]}"; do + status=0 + package_affected "${package}" "${changed}" || status=$? + case "${status}" in + 0) picked+=("${package}") ;; + 1) ;; + *) + selection="all (package graph unreadable)" + return + ;; + esac + done + SELECTED=(${picked[@]+"${picked[@]}"}) + selection="affected" +} +if [[ -n "${BASELINE_DIR}" ]]; then + select_packages + echo "Benchmark packages, ${selection}: ${SELECTED[*]:-none}" +fi + # Retain provenance without dumping credentials, remote URLs or working paths. # PR checkouts may be synthetic merges; Go selects a toolchain for each tree. metadata="${CURRENT_DIR}/bench-metadata.txt" revision() { - # An archived tree nested inside another checkout must not inherit its SHA. - local prefix - prefix="$(git -C "$1" rev-parse --show-prefix 2>/dev/null)" || { echo unavailable; return; } - [[ -z "${prefix}" ]] || { echo unavailable; return; } + is_git_root "$1" || { echo unavailable; return; } git -C "$1" rev-parse --verify "$2" 2>/dev/null || echo unavailable } { @@ -50,7 +149,8 @@ revision() { "$(date -u +%FT%TZ)" "${SAMPLE_COUNT}" "${BENCHTIME}" printf 'platform=%s\n' "$(uname -sm)" printf 'gomaxprocs=%s\n' "${GOMAXPROCS:-runtime-default}" - printf 'packages=%s\n' "${PACKAGES[*]}" + printf 'selection=%s\n' "${selection}" + printf 'packages=%s\n' "${SELECTED[*]:-}" for label in candidate baseline; do tree="${CURRENT_DIR}" [[ "${label}" != baseline ]] || tree="${BASELINE_DIR}" @@ -61,9 +161,6 @@ revision() { done } > "${metadata}" -work_dir="$(mktemp -d)" -trap 'rm -rf "${work_dir}"' EXIT - # Hash the actual executable Go is about to run, not a later reconstruction. # Keep hashing outside benchmark timing and never log binary paths or arguments. binary_wrapper="${work_dir}/record-binary" @@ -95,7 +192,7 @@ run_sample() { PULSE_DATA_DIR="${data_dir}" go test -exec "${binary_wrapper}" \ -bench=. -benchmem -count=1 -run='^$' \ -benchtime="${BENCHTIME}" -timeout=5m \ - "${PACKAGES[@]}" + "${SELECTED[@]}" ) | tee -a "${output}" } @@ -124,6 +221,30 @@ candidate_output="${CURRENT_DIR}/bench-results.txt" : > "${baseline_output}" : > "${candidate_output}" +if ((${#SELECTED[@]} == 0)); then + echo "No benchmarked package depends on a changed file; nothing to compare." + exit 0 +fi + +# Project the whole run from the rounds measured so far. Stop once it would +# reach the budget, naming each package's mean seconds per sample. +check_budget() { + local round="$1" elapsed="${SECONDS}" projected + [[ -n "${BUDGET_SECONDS}" ]] || return 0 + ((round < SAMPLE_COUNT)) || return 0 + projected=$((elapsed + (elapsed - sampling_started) * (SAMPLE_COUNT - round) / round)) + ((projected >= BUDGET_SECONDS)) || return 0 + { + echo "Benchmark budget exceeded: ${round} of ${SAMPLE_COUNT} paired rounds took ${elapsed}s" \ + "including warm-up, projecting ${projected}s against a ${BUDGET_SECONDS}s budget." + echo "Mean seconds per sample by package:" + awk '$1 == "ok" { total[$2] += $3; count[$2]++ } + END { for (p in total) printf "%8.1f %s\n", total[p] / count[p], p }' \ + "${baseline_output}" "${candidate_output}" | sort -rn + } >&2 + exit 1 +} + # Populate build and filesystem caches without adding a timed sample. This # avoids consistently charging the first measured tree for cold setup. for tree in "${BASELINE_DIR}" "${CURRENT_DIR}"; do @@ -131,10 +252,11 @@ for tree in "${BASELINE_DIR}" "${CURRENT_DIR}"; do cd "${tree}" PULSE_DATA_DIR="${work_dir}/warmup" go test \ -bench=. -count=1 -run='^$' -benchtime=1x -timeout=5m \ - "${PACKAGES[@]}" >/dev/null + "${SELECTED[@]}" >/dev/null ) done +sampling_started="${SECONDS}" for ((round = 1; round <= SAMPLE_COUNT; round++)); do if ((round % 2 == 1)); then run_sample "${BASELINE_DIR}" "${baseline_output}" baseline "${round}" @@ -143,4 +265,5 @@ for ((round = 1; round <= SAMPLE_COUNT; round++)); do run_sample "${CURRENT_DIR}" "${candidate_output}" candidate "${round}" run_sample "${BASELINE_DIR}" "${baseline_output}" baseline "${round}" fi + check_budget "${round}" done diff --git a/scripts/tests/test-ci-benchmarks.sh b/scripts/tests/test-ci-benchmarks.sh index 24747eadc7..34804e8444 100755 --- a/scripts/tests/test-ci-benchmarks.sh +++ b/scripts/tests/test-ci-benchmarks.sh @@ -22,11 +22,27 @@ if [[ "$*" == version ]]; then echo 'go version go1.26.7 linux/amd64' exit 0 fi +# Every package's closure is itself plus internal/shared, except pkg/auth. +if [[ "$1" == list ]]; then + [[ "${FAKE_GO_LIST_FAIL:-0}" == 0 ]] || exit 1 + if [[ "$2" == -m ]]; then + echo "$PWD" + exit 0 + fi + package="${!#}" + package="${package#./}" + package="${package%/}" + printf '%s/%s\n\n' "$PWD" "${package}" + [[ "${package}" == pkg/auth ]] || echo "$PWD/internal/shared" + exit 0 +fi printf '%s\t%s\n' "$PWD" "$*" >> "${FAKE_GO_LOG}" args=("$@") for ((i=0; i<${#args[@]}; i++)); do if [[ "${args[i]}" == -exec ]]; then - exec "${args[i+1]}" "${FAKE_BINARY}" '-test.bench=.' + "${args[i+1]}" "${FAKE_BINARY}" '-test.bench=.' || exit + printf 'ok \texample.test/bench\t1.500s\n' + exit 0 fi done cat <<'RESULT' @@ -65,6 +81,8 @@ mapfile -t calls < "${WORK_DIR}/go.log" metadata="${WORK_DIR}/candidate/bench-metadata.txt" grep -qFx 'samples=2' "${metadata}" +grep -qFx 'selection=all (trees are not both clean Git checkouts)' "${metadata}" +grep -qFx 'packages=./pkg/metrics/ ./pkg/auth/ ./internal/api/ ./internal/monitoring/ ./internal/unifiedresources/ ./internal/dockeragent/ ./cmd/pulse-agent/ ./internal/hostagent/ ./internal/hostmetrics/' "${metadata}" grep -qFx 'benchtime=100ms' "${metadata}" grep -qFx 'candidate.commit=unavailable' "${metadata}" grep -qFx 'baseline.go=go version go1.26.7 linux/amd64' "${metadata}" @@ -77,19 +95,83 @@ for tuple in baseline,1 candidate,1 candidate,2 baseline,2; do grep -qFx "binary=${tuple},fixture.test,${digest}" "${metadata}" done -# Real Git roots retain exact identities; a repeated run replaces old metadata. +# Real Git roots retain exact identities and measure only the packages whose +# dependency closure the candidate changes; a repeated run replaces metadata. for tree in candidate baseline; do git -C "${WORK_DIR}/${tree}" init -q - git -C "${WORK_DIR}/${tree}" -c user.name=Test -c user.email=test@example.invalid \ - commit -qm fixture --allow-empty + mkdir -p "${WORK_DIR}/${tree}/internal/shared" "${WORK_DIR}/${tree}/internal/api" + echo 'module example.test' > "${WORK_DIR}/${tree}/go.mod" + echo 'package shared' > "${WORK_DIR}/${tree}/internal/shared/shared.go" + echo 'package api' > "${WORK_DIR}/${tree}/internal/api/handler.go" done -PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \ - PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \ - PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=1 \ - bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null +change() { + mkdir -p "$(dirname "${WORK_DIR}/candidate/$1")" + echo "// $2" >> "${WORK_DIR}/candidate/$1" +} +commit() { + git -C "${WORK_DIR}/$1" add go.mod internal frontend 2>/dev/null || git -C "${WORK_DIR}/$1" add go.mod internal + git -C "${WORK_DIR}/$1" -c user.name=Test -c user.email=test@example.invalid commit -qm fixture +} +run_paired() { + : > "${WORK_DIR}/go.log" + PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \ + PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \ + PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=1 \ + bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" "$@" >/dev/null +} +commit baseline + +# Frontend files and another package's tests reach no benchmark binary. +change frontend/app.ts frontend +change internal/shared/shared_test.go shared-test +commit candidate +run_paired +grep -qFx 'selection=affected' "${metadata}" +grep -qFx 'packages=' "${metadata}" grep -qFx "candidate.commit=$(git -C "${WORK_DIR}/candidate" rev-parse HEAD)" "${metadata}" grep -qFx "baseline.tree=$(git -C "${WORK_DIR}/baseline" rev-parse 'HEAD^{tree}')" "${metadata}" +[[ "$(grep -c '^sample=' "${metadata}" || true)" == 0 ]] +[[ ! -s "${WORK_DIR}/candidate/bench-results.txt" ]] +[[ ! -s "${WORK_DIR}/candidate/bench-baseline.txt" ]] +[[ ! -s "${WORK_DIR}/go.log" ]] + +# A benchmark package's own test file selects that package alone. +change internal/api/handler_test.go api-test +commit candidate +run_paired +grep -qFx 'packages=./internal/api/' "${metadata}" [[ "$(grep -c '^sample=' "${metadata}")" == 2 ]] +[[ "$(grep -c '^BenchmarkExample' "${WORK_DIR}/candidate/bench-results.txt")" == 1 ]] +mapfile -t calls < "${WORK_DIR}/go.log" +[[ "${#calls[@]}" == 4 ]] +for call in "${calls[@]}"; do + [[ "${call}" == *' ./internal/api/' && "${call}" != *pkg/metrics* ]] +done + +# A shared non-test file selects every package that imports its directory. +change internal/shared/shared.go shared +commit candidate +run_paired +grep -qFx 'selection=affected' "${metadata}" +grep -qFx 'packages=./pkg/metrics/ ./internal/api/ ./internal/monitoring/ ./internal/unifiedresources/ ./internal/dockeragent/ ./cmd/pulse-agent/ ./internal/hostagent/ ./internal/hostmetrics/' "${metadata}" + +# An unreadable package graph measures everything rather than guessing. +FAKE_GO_LIST_FAIL=1 run_paired +grep -qFx 'selection=all (package graph unreadable)' "${metadata}" +grep -qF 'packages=./pkg/metrics/ ./pkg/auth/' "${metadata}" + +# Uncommitted edits are measured but invisible to the committed trees. +echo '// local edit' >> "${WORK_DIR}/baseline/internal/api/handler.go" +run_paired +grep -qFx 'selection=all (trees are not both clean Git checkouts)' "${metadata}" +git -C "${WORK_DIR}/baseline" checkout -q -- internal/api/handler.go + +# Module changes can move any package. +change go.mod module +commit candidate +run_paired +grep -qFx 'selection=all (module files changed)' "${metadata}" +grep -qF 'packages=./pkg/metrics/ ./pkg/auth/' "${metadata}" # A nested archive has no identity of its own; never attribute its parent's SHA. mkdir -p "${WORK_DIR}/candidate/archive" @@ -99,6 +181,7 @@ PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \ bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null grep -qFx 'candidate.commit=unavailable' "${WORK_DIR}/candidate/archive/bench-metadata.txt" ! grep -q '^baseline\.' "${WORK_DIR}/candidate/archive/bench-metadata.txt" +grep -qFx 'selection=all (no baseline)' "${WORK_DIR}/candidate/archive/bench-metadata.txt" grep -qFx "binary=candidate,unpaired,fixture.test,${digest}" \ "${WORK_DIR}/candidate/archive/bench-metadata.txt" @@ -114,6 +197,33 @@ set -e [[ "${status}" == 23 ]] grep -qFx "binary=candidate,unpaired,fixture.test,${digest}" "${metadata}" +# A run projected to reach its budget stops after the round that shows it, +# naming each package's mean cost; one inside its budget completes. +set +e +output="$(PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \ + PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \ + PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=3 \ + PULSE_BENCH_BUDGET_SECONDS=0 \ + bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" 2>&1 >/dev/null)" +status=$? +set -e +[[ "${status}" == 1 ]] +grep -qF 'Benchmark budget exceeded: 1 of 3 paired rounds' <<<"${output}" +grep -qE '^ +1\.5 example\.test/bench$' <<<"${output}" +[[ "$(grep -c '^sample=' "${metadata}")" == 2 ]] +PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \ + PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \ + PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=3 \ + PULSE_BENCH_BUDGET_SECONDS=3600 \ + bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null +[[ "$(grep -c '^sample=' "${metadata}")" == 6 ]] +set +e +PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" PULSE_BENCH_BUDGET_SECONDS=soon \ + bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null 2>&1 +status=$? +set -e +[[ "${status}" == 2 ]] + cat > "${WORK_DIR}/adequate.txt" <<'EOF' Example-4 100.0n ± 1% 111.0n ± 1% +11.00% (p=0.001 n=10) EOF