mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-23 20:25:56 +00:00
pulse-triage[bot]
be6179c2d2
Dependabot #2094, #2095 and #2096 bump actions/setup-go 6.4.0 -> 7.0.0, signpath/github-action-submit-signing-request 2 -> 3.0 and actions/github-script 8.0.0 -> 9.0.0. Each proposal is blocked only by the reviewed pin constants, the workflow-trust allowlist, the release-consumer action manifest and the installer governance assertions that hard-code the previous revisions. Carry all three bumps with those artifacts in one commit so the proposals can be closed as superseded. All three actions keep the node24 runtime and their existing inputs and outputs; no consumer interface, installer behaviour or public contract changes. The deployment-installability and agent-lifecycle workflow references are pin-only. Contract-Neutral: Reviewed action pin refresh with identical node24 consumer interfaces and unchanged inputs/outputs; no public-contract or installer-behaviour delta. Change-source: pulse-maintainer
92 lines
3.2 KiB
YAML
92 lines
3.2 KiB
YAML
name: Pulse AI Model Matrix
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
scenario:
|
|
description: Scenario or collection to run (e.g. matrix, smoke, readonly, advanced)
|
|
required: true
|
|
default: matrix
|
|
models:
|
|
description: Comma-separated model list (e.g. gpt-4.1-mini,claude-3-5-sonnet,gemini-1.5-pro,ollama:llama3.1)
|
|
required: false
|
|
default: ""
|
|
providers:
|
|
description: Optional provider filter (e.g. openai,anthropic,gemini,ollama)
|
|
required: false
|
|
default: ""
|
|
base_url:
|
|
description: Pulse API base URL reachable from the GitHub-hosted runner. Add Tailscale or another reachability layer to the workflow if pointing at a private instance.
|
|
required: true
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
eval:
|
|
name: Model Matrix Eval
|
|
# Moved off self-hosted: a self-hosted runner attached to a public repo
|
|
# is RCE-equivalent on the runner host as soon as one workflow gets
|
|
# mis-configured to run untrusted PR code. Reachability to the eval
|
|
# target is now the dispatcher's responsibility (pass a base_url the
|
|
# runner can reach, or add tailscale/* steps before this job).
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 60
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
|
with:
|
|
go-version-file: go.mod
|
|
cache: false
|
|
|
|
- name: Verify eval credentials are configured
|
|
env:
|
|
PULSE_EVAL_USER: ${{ secrets.PULSE_EVAL_USER }}
|
|
PULSE_EVAL_PASS: ${{ secrets.PULSE_EVAL_PASS }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ -z "${PULSE_EVAL_USER}" ] || [ -z "${PULSE_EVAL_PASS}" ]; then
|
|
echo "::error::PULSE_EVAL_USER and PULSE_EVAL_PASS repo secrets must be set."
|
|
echo "::error::Refusing to fall back to admin/admin defaults."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Run eval matrix
|
|
env:
|
|
EVAL_REPORT_DIR: tmp/eval-reports
|
|
PULSE_EVAL_USER: ${{ secrets.PULSE_EVAL_USER }}
|
|
PULSE_EVAL_PASS: ${{ secrets.PULSE_EVAL_PASS }}
|
|
INPUT_SCENARIO: ${{ inputs.scenario }}
|
|
INPUT_MODELS: ${{ inputs.models }}
|
|
INPUT_PROVIDERS: ${{ inputs.providers }}
|
|
INPUT_BASE_URL: ${{ inputs.base_url }}
|
|
run: |
|
|
set -euo pipefail
|
|
MODEL_ARGS=("-auto-models")
|
|
if [ -n "${INPUT_MODELS}" ]; then
|
|
MODEL_ARGS=("-models" "${INPUT_MODELS}")
|
|
fi
|
|
if [ -n "${INPUT_PROVIDERS}" ]; then
|
|
export EVAL_MODEL_PROVIDERS="${INPUT_PROVIDERS}"
|
|
fi
|
|
go run ./cmd/eval \
|
|
-scenario "${INPUT_SCENARIO}" \
|
|
"${MODEL_ARGS[@]}" \
|
|
-url "${INPUT_BASE_URL}" \
|
|
-user "${PULSE_EVAL_USER}" \
|
|
-pass "${PULSE_EVAL_PASS}"
|
|
|
|
- name: Upload eval reports
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: eval-reports
|
|
path: tmp/eval-reports
|
|
retention-days: 14
|