spinifex/.github/workflows/perf-3node.yml

name: Predastore 3-Node Performance

on:
  workflow_dispatch:
    inputs:
      perf_preset:
        description: "Warp workload preset: smoke (30s) or compare (2m)"
        required: false
        default: compare
        type: choice
        options:
          - smoke
          - compare
      cluster_ref:
        description: "Git ref to build the cluster from, e.g. v1.18.0. Empty uses this branch."
        required: false
        default: ""
        type: string

# Shared with e2e.yml and e2e-nightly.yml: this job provisions on the same
# hypervisors they do, so a run in flight there and here would compete for
# disks and libvirt guests. Queued, not cancelled, same as those workflows.
concurrency:
  group: e2e-hypervisors
  cancel-in-progress: false

permissions:
  contents: read

env:
  GIT_BRANCH: ${{ github.ref_name }}
  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}

jobs:
  perf_3node:
    name: Predastore Throughput (3-node)
    # ci-multi is a systemd runner on the LAN, not the ephemeral pool. A pool
    # VM sits behind the geneve overlay at MTU 1408, which would put the
    # runner's own network path inside the thing being measured.
    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 90
    env:
      TOFU_LOG: /tmp/e2e-tofu-perf3node-${{ github.run_id }}.log
      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=perf-3node
    steps:
      - name: Reap Orphan Procs
        run: |
          set +e
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 3
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          true

      - name: Clean Workspace
        run: |
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      # ci-multi and ci-single resolve to the same physical host, so the
      # nvme path cell-23 uses for its throughput run is already there.
      # Pointing TMPDIR at it keeps warp's payloads off a small /tmp tmpfs.
      - name: Configure work directories
        run: |
          E2E_ROOT=/mnt/nvme/e2e
          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"

      - uses: actions/create-github-app-token@v3.3.0
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - uses: actions/checkout@v7.0.1
        with:
          path: spinifex

      - uses: actions/setup-go@v7.0.0
        with:
          go-version-file: spinifex/go.mod
          cache: false

      - name: Re-enable Go toolchain auto-download
        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"

      # Same-named branch when one exists, dev otherwise. Matches how the
      # e2e path picks siblings, so this measures the same predastore
      # commit a cross-repo change would be built and tested against.
      - name: Resolve sibling refs
        id: siblings
        env:
          BRANCH: ${{ env.GIT_BRANCH }}
        run: bash .github/resolve-sibling-refs.sh
        working-directory: spinifex

      - uses: actions/checkout@v7.0.1
        id: predastore
        with:
          repository: mulgadc/predastore
          ref: ${{ steps.siblings.outputs.predastore }}
          path: predastore

      # The cluster's ref is a knob and the harness's is not. Measuring an old
      # release with a new harness is the only way its number can be set beside
      # a current one; rebuilding both would change the ruler and the thing.
      - name: Build Install Artifacts
        working-directory: mulga/scripts/tofu-cluster
        env:
          CLUSTER_REF: ${{ inputs.cluster_ref }}
        run: |
          ./build-install-artifacts.sh "${CLUSTER_REF:-$GIT_BRANCH}" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-tests.tar" \
            --skip-e2e-build

      # The libvirt provider dials the hypervisors by bare hostname. A pool VM
      # sits inside a VPC and cannot resolve them; a LAN runner already can, so
      # this is self-gating on the lookup rather than on the runner's labels.
      - name: Map hypervisor names
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          if getent hosts banksia >/dev/null; then
            echo "hypervisor names already resolve"
          else
            sudo tee -a /etc/hosts < network/hosts >/dev/null
            echo "added $(grep -c '^[0-9]' network/hosts) entries to /etc/hosts"
          fi
          # Pinned rather than StrictHostKeyChecking=no: these hops reach the
          # hypervisors themselves, so an unverified key is the one place in
          # this job where accepting anything would matter.
          if ssh-keygen -F banksia >/dev/null 2>&1; then
            echo "hypervisor host keys already known"
          else
            install -d -m 700 ~/.ssh
            cat network/known_hosts >> ~/.ssh/known_hosts
            chmod 600 ~/.ssh/known_hosts
            echo "pinned $(grep -c '^[a-z]' network/known_hosts) host keys"
          fi

      - name: Provision VMs
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          export TF_VAR_git_branch="${GIT_BRANCH}"
          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          ok "3-node cluster provisioned"

      - name: Bootstrap via Binary Install
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Resolve cluster SSH details
        id: cluster
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
          REGION=$(echo "$INVENTORY" | jq -r '.region')
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          NODES_CSV=""
          for i in $(seq 0 $((NODE_COUNT - 1))); do
            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
          done
          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
          echo "ssh_user=${SSH_USER}" >> "$GITHUB_OUTPUT"
          echo "region=${REGION}" >> "$GITHUB_OUTPUT"
          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"

      # PERF_EXTERNAL_HOSTS wants host:port, and the gate listens on 8443 on
      # every node. run-info.txt from this run is then comparable with the
      # nightly's loopback one, since the harness itself never changes.
      - name: Resolve PERF_EXTERNAL_HOSTS
        id: perf_hosts
        run: |
          set -euo pipefail
          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
          HOSTS=""
          for ip in "${NODE_LIST[@]}"; do
            [ -z "$ip" ] && continue
            if [ -z "$HOSTS" ]; then HOSTS="$ip:8443"; else HOSTS="$HOSTS,$ip:8443"; fi
          done
          echo "hosts=${HOSTS}" >> "$GITHUB_OUTPUT"

      # A slow bootstrap should fail here, legibly, rather than inside warp
      # with a wall of connection-refused errors. Bounded at 5 minutes per
      # gate so a genuinely dead node still fails the job in finite time.
      - name: Wait for gates to answer on :8443
        run: |
          set -euo pipefail
          IFS=',' read -ra GATES <<< "${{ steps.perf_hosts.outputs.hosts }}"
          for gate in "${GATES[@]}"; do
            echo "waiting for https://$gate"
            up=0
            for i in $(seq 1 60); do
              if curl -k -sS --max-time 5 -o /dev/null "https://$gate/"; then
                up=1
                break
              fi
              sleep 5
            done
            [ "$up" -eq 1 ] || { echo "::error::gate $gate never answered HTTPS on :8443"; exit 1; }
            echo "gate $gate is up"
          done

      # Warp reads whole objects, so no workload here touches a ranged read and
      # the throughput figures stay healthy while ranged GETs are broken. This
      # probe is the only thing in CI that looks at them.
      - name: Ranged GET probe
        env:
          GATE: ${{ steps.cluster.outputs.node1_ip }}
          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
        run: |
          set -euo pipefail
          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
          AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
          SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
          echo "::add-mask::$SK"
          B="ranged-probe-${GITHUB_RUN_ID}"
          AWSX=(--region "$CLUSTER_REGION" --endpoint-url "https://${GATE}:8443" --no-verify-ssl)
          export AWS_ACCESS_KEY_ID="$AK" AWS_SECRET_ACCESS_KEY="$SK"
          aws s3 mb "s3://$B" "${AWSX[@]}"
          trap 'aws s3 rb "s3://$B" --force "${AWSX[@]}" >/dev/null 2>&1 || true' EXIT
          # Two sizes: read amplification makes a boundary-crossing range cost
          # the whole object, so the fault shows as time tracking object size.
          for mb in 64 256; do
            head -c $((mb*1024*1024)) /dev/urandom > "${TMPDIR}/probe-${mb}.bin"
            aws s3 cp "${TMPDIR}/probe-${mb}.bin" "s3://$B/probe-${mb}.bin" --only-show-errors "${AWSX[@]}"
            rm -f "${TMPDIR}/probe-${mb}.bin"
          done
          t() {
            curl -sk --aws-sigv4 "aws:amz:${CLUSTER_REGION}:s3" -u "$AK:$SK" \
              -H "Range: bytes=0-$1" -o /dev/null -w '%{time_total}' \
              --url "https://${GATE}:8443/${B}/probe-${2}.bin"
          }
          # 4194304 is the last byte inside the first 4 MiB block; one more
          # crosses into the second and leaves the single-shard fast path.
          IN64=$(t 4194303 64);  OUT64=$(t 4194304 64)
          IN256=$(t 4194303 256); OUT256=$(t 4194304 256)
          printf '%-10s in-block=%ss  crossing=%ss\n' "64MiB" "$IN64" "$OUT64"
          printf '%-10s in-block=%ss  crossing=%ss\n' "256MiB" "$IN256" "$OUT256"
          # The 4x object-size ratio is the signal, and it is compared against
          # the crossing reads alone so a slow runner cancels out of both sides.
          awk -v a="$OUT64" -v b="$OUT256" 'BEGIN{
            r = (a > 0) ? b/a : 0
            printf "crossing-range cost ratio 256MiB/64MiB = %.2f (object size ratio is 4.00)\n", r
            if (r > 2.0) {
              print "::error::a boundary-crossing ranged GET scales with object size, so it is reading the whole object"
              exit 1
            }
          }'
          echo "ranged GET does not scale with object size"

      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
      # free-space watermark, so this asserts the same headroom cell-23 does.
      # tmpfs is rejected outright: that would measure RAM, not the gates.
      - name: Preflight
        run: |
          set -euo pipefail
          mkdir -p "$TMPDIR"
          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
          case "$fstype" in
            tmpfs|ramfs)
              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
              exit 1 ;;
          esac
          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
          df -h "$TMPDIR"
          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
            exit 1
          fi

      - name: Install warp
        working-directory: predastore
        run: make warp-install

      # PERF_EXTERNAL_HOSTS skips profile selection, port offsetting, and
      # cluster start/stop, and measures these three gates directly. Same
      # workloads and analysis as the nightly loopback run, so the numbers
      # from the two are directly comparable.
      #
      # The cluster's credentials are read and used inside this one step, the
      # way e2e-ddil.yml does it: a step-level env: block is echoed verbatim
      # into the run log, and a shell variable is not.
      - name: Run 3-node throughput benchmark
        working-directory: predastore
        env:
          PERF_EXTERNAL_HOSTS: ${{ steps.perf_hosts.outputs.hosts }}
          PERF_CONFIGS: spinifex-3node
          PERF_PRESET: ${{ inputs.perf_preset || 'compare' }}
          # What is deployed, not what the harness was checked out at. With a
          # cluster_ref those differ, and run-info.txt records the cluster.
          PERF_EXTERNAL_SHA: ${{ inputs.cluster_ref || steps.predastore.outputs.commit }}
          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
        run: |
          set -euo pipefail
          # spx admin init mints a random pair at bootstrap and writes it to
          # node1's ~/.aws/credentials. The benchmark's built-in key is the one
          # the loopback configs trust, which this cluster has never heard of.
          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
          PERF_ACCESS_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
          PERF_SECRET_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
          if [ -z "$PERF_ACCESS_KEY" ] || [ -z "$PERF_SECRET_KEY" ]; then
            echo "::error::could not read spinifex AWS credentials from ${NODE1_IP}"
            exit 1
          fi
          echo "::add-mask::$PERF_SECRET_KEY"
          mkdir -p "$PERF_RESULTS_ROOT"
          PERF_ACCESS_KEY="$PERF_ACCESS_KEY" \
          PERF_SECRET_KEY="$PERF_SECRET_KEY" \
          PERF_REGION="$CLUSTER_REGION" \
            make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"

      # Report-only, matching cell-23: the baseline comparison and any
      # regression gate are future work once run-to-run spread is known from
      # real bare-metal runs.
      - uses: actions/upload-artifact@v7.0.2
        if: always()
        with:
          name: predastore-perf-3node-${{ github.run_id }}
          path: ${{ env.PERF_RESULTS_ROOT }}
          if-no-files-found: warn
          retention-days: 90

      - name: Clean up results
        if: always()
        run: rm -rf "${PERF_RESULTS_ROOT}"

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set +e
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 2
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

      - name: Upload tofu log
        if: always()
        uses: actions/upload-artifact@v7.0.2
        with:
          name: tofu-log-perf3node-${{ github.run_id }}
          path: ${{ env.TOFU_LOG }}
          if-no-files-found: ignore
          retention-days: 14