diff --git a/scripts/BENCHMARK_EVIDENCE.md b/scripts/BENCHMARK_EVIDENCE.md index 0a584eadb..8f2d69d1e 100644 --- a/scripts/BENCHMARK_EVIDENCE.md +++ b/scripts/BENCHMARK_EVIDENCE.md @@ -7,8 +7,8 @@ selected Go versions, platform, GOMAXPROCS setting, sample count, duration, package list and paired sample start order/timestamps. It deliberately does not dump environment variables, remote URLs or filesystem paths. -HEAD/tree identify committed source, not a clean-worktree attestation or a -binary digest. The workflow adds a frontend embed stub. A PR checkout may be +HEAD/tree identify committed source, not a clean-worktree or clean-build +attestation. The workflow adds a frontend embed stub. A PR checkout may be a synthetic merge rather than the PR head; use the captured identity, not the run API's head SHA alone, when reproducing it. Non-Git trees and nested archives report identity unavailable rather than inheriting an enclosing checkout's SHA. @@ -24,3 +24,17 @@ Unchanged source in a benchmark's file does not establish unchanged generated code, binary layout or runtime behaviour. Matching address-normalised instructions alone do not prove equivalent timing. Do not waive the gate from an unrelated diff, a single passing repeat or an unverified noise hypothesis. + +Each measured Go test executable is now SHA-256 hashed immediately before +execution through Go's `-exec` wrapper. Metadata `binary=label,round,name,sha256` +records bind each paired ordinal to the executed bytes; unpaired runs use +`unpaired` because Go performs all counts in one invocation. Warmups are not +recorded. Hashing is outside the benchmark timer, but it touches executable +pages and is an instrumentation change, not a controlled comparison with older +runs. Execution arguments, temporary paths and environment contents are not +recorded. A failing executable still fails collection. + +These hashes do not retain binaries or establish equivalent instructions, +linked data, host state or performance. They make later reconstructions +checkable; a hash mismatch means they are not the original measured bytes. +Historical failures without hashes remain unresolved evidence. diff --git a/scripts/run-ci-benchmarks.sh b/scripts/run-ci-benchmarks.sh index 8082c5c24..12e3fe0fa 100755 --- a/scripts/run-ci-benchmarks.sh +++ b/scripts/run-ci-benchmarks.sh @@ -64,6 +64,21 @@ revision() { work_dir="$(mktemp -d)" trap 'rm -rf "${work_dir}"' EXIT +# Hash the actual executable Go is about to run, not a later reconstruction. +# Keep hashing outside benchmark timing and never log binary paths or arguments. +binary_wrapper="${work_dir}/record-binary" +cat > "${binary_wrapper}" <<'WRAPPER' +#!/usr/bin/env bash +set -euo pipefail +digest="$(sha256sum -- "$1")" +digest="${digest%% *}" +printf 'binary=%s,%s,%s,%s\n' "${PULSE_BENCH_LABEL}" \ + "${PULSE_BENCH_ROUND}" "${1##*/}" "${digest}" >> "${PULSE_BENCH_METADATA}" +exec "$@" +WRAPPER +chmod +x "${binary_wrapper}" +export PULSE_BENCH_METADATA="${metadata}" + run_sample() { local tree="$1" local output="$2" @@ -76,7 +91,8 @@ run_sample() { echo "=== ${label} benchmark sample ${round}/${SAMPLE_COUNT} ===" ( cd "${tree}" - PULSE_DATA_DIR="${data_dir}" go test \ + PULSE_BENCH_LABEL="${label}" PULSE_BENCH_ROUND="${round}" \ + PULSE_DATA_DIR="${data_dir}" go test -exec "${binary_wrapper}" \ -bench=. -benchmem -count=1 -run='^$' \ -benchtime="${BENCHTIME}" -timeout=5m \ "${PACKAGES[@]}" @@ -90,7 +106,8 @@ run_unpaired() { mkdir -p "${data_dir}" ( cd "${CURRENT_DIR}" - PULSE_DATA_DIR="${data_dir}" go test \ + PULSE_BENCH_LABEL=candidate PULSE_BENCH_ROUND=unpaired \ + PULSE_DATA_DIR="${data_dir}" go test -exec "${binary_wrapper}" \ -bench=. -benchmem -count="${SAMPLE_COUNT}" -run='^$' \ -benchtime="${BENCHTIME}" -timeout=5m \ "${PACKAGES[@]}" diff --git a/scripts/tests/test-ci-benchmarks.sh b/scripts/tests/test-ci-benchmarks.sh index 01dabe584..24747eadc 100755 --- a/scripts/tests/test-ci-benchmarks.sh +++ b/scripts/tests/test-ci-benchmarks.sh @@ -8,6 +8,14 @@ WORK_DIR="$(mktemp -d)" trap 'rm -rf "${WORK_DIR}"' EXIT mkdir -p "${WORK_DIR}/bin" "${WORK_DIR}/candidate" "${WORK_DIR}/baseline" +cat > "${WORK_DIR}/bin/fixture.test" <<'EOF' +#!/usr/bin/env bash +[[ "$1" == '-test.bench=.' ]] || exit 91 +[[ "${FAKE_BINARY_FAIL:-0}" == 0 ]] || exit 23 +echo 'BenchmarkExample-4 1 100 ns/op 0 B/op 0 allocs/op' +EOF +chmod +x "${WORK_DIR}/bin/fixture.test" +export FAKE_BINARY="${WORK_DIR}/bin/fixture.test" cat > "${WORK_DIR}/bin/go" <<'EOF' #!/usr/bin/env bash if [[ "$*" == version ]]; then @@ -15,6 +23,12 @@ if [[ "$*" == version ]]; then exit 0 fi printf '%s\t%s\n' "$PWD" "$*" >> "${FAKE_GO_LOG}" +args=("$@") +for ((i=0; i<${#args[@]}; i++)); do + if [[ "${args[i]}" == -exec ]]; then + exec "${args[i+1]}" "${FAKE_BINARY}" '-test.bench=.' + fi +done cat <<'RESULT' goos: linux goarch: amd64 @@ -56,6 +70,12 @@ grep -qFx 'candidate.commit=unavailable' "${metadata}" grep -qFx 'baseline.go=go version go1.26.7 linux/amd64' "${metadata}" [[ "$(grep -c '^sample=' "${metadata}")" == 4 ]] ! grep -qF "${WORK_DIR}" "${metadata}" +digest="$(sha256sum "${FAKE_BINARY}")" +digest="${digest%% *}" +[[ "$(grep -c '^binary=' "${metadata}")" == 4 ]] +for tuple in baseline,1 candidate,1 candidate,2 baseline,2; do + grep -qFx "binary=${tuple},fixture.test,${digest}" "${metadata}" +done # Real Git roots retain exact identities; a repeated run replaces old metadata. for tree in candidate baseline; do @@ -80,6 +100,20 @@ PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \ grep -qFx 'candidate.commit=unavailable' "${WORK_DIR}/candidate/archive/bench-metadata.txt" ! grep -q '^baseline\.' "${WORK_DIR}/candidate/archive/bench-metadata.txt" +grep -qFx "binary=candidate,unpaired,fixture.test,${digest}" \ + "${WORK_DIR}/candidate/archive/bench-metadata.txt" + +# The wrapper must not turn a failed executable into successful evidence. +set +e +PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \ + FAKE_BINARY_FAIL=1 PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \ + PULSE_BENCH_BASELINE_DIR='' PULSE_BENCH_SAMPLE_COUNT=1 \ + bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null +status=$? +set -e +[[ "${status}" == 23 ]] +grep -qFx "binary=candidate,unpaired,fixture.test,${digest}" "${metadata}" + cat > "${WORK_DIR}/adequate.txt" <<'EOF' Example-4 100.0n ± 1% 111.0n ± 1% +11.00% (p=0.001 n=10) EOF