fix(ci): bind benchmark samples to executed binary hashes

Record executable SHA-256 before measured Go test invocations without logging paths or arguments. Preserve sample ordering and failure status; document instrumentation limits rather than dismissing historical regressions.

Change-source: pulse-maintainer

scripts/BENCHMARK_EVIDENCE.md

@@ -7,8 +7,8 @@ selected Go versions, platform, GOMAXPROCS setting, sample count, duration,
 package list and paired sample start order/timestamps. It deliberately does
 not dump environment variables, remote URLs or filesystem paths.
 
-HEAD/tree identify committed source, not a clean-worktree attestation or a
-binary digest. The workflow adds a frontend embed stub. A PR checkout may be
+HEAD/tree identify committed source, not a clean-worktree or clean-build
+attestation. The workflow adds a frontend embed stub. A PR checkout may be
 a synthetic merge rather than the PR head; use the captured identity, not the
 run API's head SHA alone, when reproducing it. Non-Git trees and nested archives
 report identity unavailable rather than inheriting an enclosing checkout's SHA.
@@ -24,3 +24,17 @@ Unchanged source in a benchmark's file does not establish unchanged generated
 code, binary layout or runtime behaviour. Matching address-normalised
 instructions alone do not prove equivalent timing. Do not waive the gate from
 an unrelated diff, a single passing repeat or an unverified noise hypothesis.
+
+Each measured Go test executable is now SHA-256 hashed immediately before
+execution through Go's `-exec` wrapper. Metadata `binary=label,round,name,sha256`
+records bind each paired ordinal to the executed bytes; unpaired runs use
+`unpaired` because Go performs all counts in one invocation. Warmups are not
+recorded. Hashing is outside the benchmark timer, but it touches executable
+pages and is an instrumentation change, not a controlled comparison with older
+runs. Execution arguments, temporary paths and environment contents are not
+recorded. A failing executable still fails collection.
+
+These hashes do not retain binaries or establish equivalent instructions,
+linked data, host state or performance. They make later reconstructions
+checkable; a hash mismatch means they are not the original measured bytes.
+Historical failures without hashes remain unresolved evidence.

scripts/run-ci-benchmarks.sh

@@ -64,6 +64,21 @@ revision() {
 work_dir="$(mktemp -d)"
 trap 'rm -rf "${work_dir}"' EXIT
 
+# Hash the actual executable Go is about to run, not a later reconstruction.
+# Keep hashing outside benchmark timing and never log binary paths or arguments.
+binary_wrapper="${work_dir}/record-binary"
+cat > "${binary_wrapper}" <<'WRAPPER'
+#!/usr/bin/env bash
+set -euo pipefail
+digest="$(sha256sum -- "$1")"
+digest="${digest%% *}"
+printf 'binary=%s,%s,%s,%s\n' "${PULSE_BENCH_LABEL}" \
+  "${PULSE_BENCH_ROUND}" "${1##*/}" "${digest}" >> "${PULSE_BENCH_METADATA}"
+exec "$@"
+WRAPPER
+chmod +x "${binary_wrapper}"
+export PULSE_BENCH_METADATA="${metadata}"
+
 run_sample() {
   local tree="$1"
   local output="$2"
@@ -76,7 +91,8 @@ run_sample() {
   echo "=== ${label} benchmark sample ${round}/${SAMPLE_COUNT} ==="
   (
     cd "${tree}"
-    PULSE_DATA_DIR="${data_dir}" go test \
+    PULSE_BENCH_LABEL="${label}" PULSE_BENCH_ROUND="${round}" \
+    PULSE_DATA_DIR="${data_dir}" go test -exec "${binary_wrapper}" \
       -bench=. -benchmem -count=1 -run='^$' \
       -benchtime="${BENCHTIME}" -timeout=5m \
       "${PACKAGES[@]}"
@@ -90,7 +106,8 @@ run_unpaired() {
   mkdir -p "${data_dir}"
   (
     cd "${CURRENT_DIR}"
-    PULSE_DATA_DIR="${data_dir}" go test \
+    PULSE_BENCH_LABEL=candidate PULSE_BENCH_ROUND=unpaired \
+    PULSE_DATA_DIR="${data_dir}" go test -exec "${binary_wrapper}" \
       -bench=. -benchmem -count="${SAMPLE_COUNT}" -run='^$' \
       -benchtime="${BENCHTIME}" -timeout=5m \
       "${PACKAGES[@]}"

scripts/tests/test-ci-benchmarks.sh

@@ -8,6 +8,14 @@ WORK_DIR="$(mktemp -d)"
 trap 'rm -rf "${WORK_DIR}"' EXIT
 
 mkdir -p "${WORK_DIR}/bin" "${WORK_DIR}/candidate" "${WORK_DIR}/baseline"
+cat > "${WORK_DIR}/bin/fixture.test" <<'EOF'
+#!/usr/bin/env bash
+[[ "$1" == '-test.bench=.' ]] || exit 91
+[[ "${FAKE_BINARY_FAIL:-0}" == 0 ]] || exit 23
+echo 'BenchmarkExample-4  1  100 ns/op  0 B/op  0 allocs/op'
+EOF
+chmod +x "${WORK_DIR}/bin/fixture.test"
+export FAKE_BINARY="${WORK_DIR}/bin/fixture.test"
 cat > "${WORK_DIR}/bin/go" <<'EOF'
 #!/usr/bin/env bash
 if [[ "$*" == version ]]; then
@@ -15,6 +23,12 @@ if [[ "$*" == version ]]; then
   exit 0
 fi
 printf '%s\t%s\n' "$PWD" "$*" >> "${FAKE_GO_LOG}"
+args=("$@")
+for ((i=0; i<${#args[@]}; i++)); do
+  if [[ "${args[i]}" == -exec ]]; then
+    exec "${args[i+1]}" "${FAKE_BINARY}" '-test.bench=.'
+  fi
+done
 cat <<'RESULT'
 goos: linux
 goarch: amd64
@@ -56,6 +70,12 @@ grep -qFx 'candidate.commit=unavailable' "${metadata}"
 grep -qFx 'baseline.go=go version go1.26.7 linux/amd64' "${metadata}"
 [[ "$(grep -c '^sample=' "${metadata}")" == 4 ]]
 ! grep -qF "${WORK_DIR}" "${metadata}"
+digest="$(sha256sum "${FAKE_BINARY}")"
+digest="${digest%% *}"
+[[ "$(grep -c '^binary=' "${metadata}")" == 4 ]]
+for tuple in baseline,1 candidate,1 candidate,2 baseline,2; do
+  grep -qFx "binary=${tuple},fixture.test,${digest}" "${metadata}"
+done
 
 # Real Git roots retain exact identities; a repeated run replaces old metadata.
 for tree in candidate baseline; do
@@ -80,6 +100,20 @@ PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
 grep -qFx 'candidate.commit=unavailable' "${WORK_DIR}/candidate/archive/bench-metadata.txt"
 ! grep -q '^baseline\.' "${WORK_DIR}/candidate/archive/bench-metadata.txt"
 
+grep -qFx "binary=candidate,unpaired,fixture.test,${digest}" \
+  "${WORK_DIR}/candidate/archive/bench-metadata.txt"
+
+# The wrapper must not turn a failed executable into successful evidence.
+set +e
+PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
+  FAKE_BINARY_FAIL=1 PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
+  PULSE_BENCH_BASELINE_DIR='' PULSE_BENCH_SAMPLE_COUNT=1 \
+  bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
+status=$?
+set -e
+[[ "${status}" == 23 ]]
+grep -qFx "binary=candidate,unpaired,fixture.test,${digest}" "${metadata}"
+
 cat > "${WORK_DIR}/adequate.txt" <<'EOF'
 Example-4  100.0n ± 1%  111.0n ± 1%  +11.00% (p=0.001 n=10)
 EOF