pulse/.github/workflows/eval-model-matrix.yml

name: Pulse AI Model Matrix

on:
  workflow_dispatch:
    inputs:
      scenario:
        description: Scenario or collection to run (e.g. matrix, smoke, readonly, advanced)
        required: true
        default: matrix
      models:
        description: Comma-separated model list (e.g. gpt-4.1-mini,claude-3-5-sonnet,gemini-1.5-pro,ollama:llama3.1)
        required: false
        default: ""
      providers:
        description: Optional provider filter (e.g. openai,anthropic,gemini,ollama)
        required: false
        default: ""
      base_url:
        description: Pulse API base URL reachable from the GitHub-hosted runner. Add Tailscale or another reachability layer to the workflow if pointing at a private instance.
        required: true

permissions:
  contents: read

jobs:
  eval:
    name: Model Matrix Eval
    # Moved off self-hosted: a self-hosted runner attached to a public repo
    # is RCE-equivalent on the runner host as soon as one workflow gets
    # mis-configured to run untrusted PR code. Reachability to the eval
    # target is now the dispatcher's responsibility (pass a base_url the
    # runner can reach, or add tailscale/* steps before this job).
    runs-on: ubuntu-24.04
    timeout-minutes: 60

    steps:
      - name: Checkout code
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false

      - name: Set up Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version-file: go.mod

      - name: Verify eval credentials are configured
        env:
          PULSE_EVAL_USER: ${{ secrets.PULSE_EVAL_USER }}
          PULSE_EVAL_PASS: ${{ secrets.PULSE_EVAL_PASS }}
        run: |
          set -euo pipefail
          if [ -z "${PULSE_EVAL_USER}" ] || [ -z "${PULSE_EVAL_PASS}" ]; then
            echo "::error::PULSE_EVAL_USER and PULSE_EVAL_PASS repo secrets must be set."
            echo "::error::Refusing to fall back to admin/admin defaults."
            exit 1
          fi

      - name: Run eval matrix
        env:
          EVAL_REPORT_DIR: tmp/eval-reports
          PULSE_EVAL_USER: ${{ secrets.PULSE_EVAL_USER }}
          PULSE_EVAL_PASS: ${{ secrets.PULSE_EVAL_PASS }}
          INPUT_SCENARIO: ${{ inputs.scenario }}
          INPUT_MODELS: ${{ inputs.models }}
          INPUT_PROVIDERS: ${{ inputs.providers }}
          INPUT_BASE_URL: ${{ inputs.base_url }}
        run: |
          set -euo pipefail
          MODEL_ARGS=("-auto-models")
          if [ -n "${INPUT_MODELS}" ]; then
            MODEL_ARGS=("-models" "${INPUT_MODELS}")
          fi
          if [ -n "${INPUT_PROVIDERS}" ]; then
            export EVAL_MODEL_PROVIDERS="${INPUT_PROVIDERS}"
          fi
          go run ./cmd/eval \
            -scenario "${INPUT_SCENARIO}" \
            "${MODEL_ARGS[@]}" \
            -url "${INPUT_BASE_URL}" \
            -user "${PULSE_EVAL_USER}" \
            -pass "${PULSE_EVAL_PASS}"

      - name: Upload eval reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval-reports
          path: tmp/eval-reports
          retention-days: 14