Preserve coding-plan provenance in qualification

This commit is contained in:
rcourtman
2026-07-15 00:06:56 +01:00
parent 61ef640a51
commit a8e42fc7fe
8 changed files with 62 additions and 1 deletions
+6
View File
@@ -415,6 +415,12 @@ is not represented as a zero-dollar API price. The report keeps its monetary
cost unknown and marks the per-run metered-API budget as not applicable; plan
limits, provider errors, latency, and any usage the CLI exposes remain visible.
Z.ai requests sent through a configured `/api/coding/paas/` endpoint are
recorded as `inference_route=coding_plan_allowance`. Qualification keeps their
per-run monetary cost unknown and the metered-API dollar budget not applicable,
while still scoring tokens, latency, provider or plan failures, and model
quality. The standard Z.ai `/api/paas/` endpoint remains a `metered_api` route.
### Models
Pulse uses model identifiers in the form: `provider:model-name`
@@ -297,6 +297,9 @@ pricing on `metered_api` remains a fail-closed dollar-budget failure. A
monetary cost as unknown with `budget_applicable=false`; it must not invent a
zero-dollar price, and latency, provider/plan errors, tool efficiency, and any
usage counts exposed by the transport remain scored and reportable.
The configured Z.ai `/api/coding/paas/` endpoint is likewise reported as
`coding_plan_allowance`, with unknown monetary cost and no per-run metered-API
dollar budget; the standard `/api/paas/` endpoint remains `metered_api`.
## Canonical Files
+1
View File
@@ -54,6 +54,7 @@ type AISettings struct {
PatrolEnabled bool `json:"patrol_enabled"`
CodexSubscriptionEnabled bool `json:"codex_subscription_enabled"`
ClaudeSubscriptionEnabled bool `json:"claude_subscription_enabled"`
ZaiBaseURL string `json:"zai_base_url"`
}
type PulseVersion struct {
+13
View File
@@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"html"
"net/url"
"os"
"path/filepath"
"regexp"
@@ -52,6 +53,18 @@ func inferenceRouteForProvider(provider string) string {
}
}
func inferenceRouteForProviderEndpoint(provider, baseURL string) string {
provider = strings.ToLower(strings.TrimSpace(provider))
if provider != "zai" {
return inferenceRouteForProvider(provider)
}
parsed, err := url.Parse(strings.TrimSpace(baseURL))
if err == nil && strings.Contains(strings.ToLower(parsed.Path), "/coding/paas/") {
return "coding_plan_allowance"
}
return inferenceRouteForProvider(provider)
}
type RunReport struct {
SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"`
@@ -0,0 +1,25 @@
package qualification
import "testing"
func TestInferenceRouteForProviderEndpoint(t *testing.T) {
tests := []struct {
name string
provider string
baseURL string
want string
}{
{name: "Zai coding plan", provider: "zai", baseURL: "https://api.z.ai/api/coding/paas/v4", want: "coding_plan_allowance"},
{name: "Zai coding completions", provider: "ZAI", baseURL: "https://api.z.ai/api/coding/paas/v4/chat/completions", want: "coding_plan_allowance"},
{name: "Zai metered API", provider: "zai", baseURL: "https://api.z.ai/api/paas/v4", want: "metered_api"},
{name: "unparseable Zai endpoint", provider: "zai", baseURL: "://", want: "metered_api"},
{name: "local subscription agent", provider: "codex-subscription", baseURL: "", want: "local_subscription_agent"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := inferenceRouteForProviderEndpoint(tt.provider, tt.baseURL); got != tt.want {
t.Fatalf("inferenceRouteForProviderEndpoint(%q, %q) = %q, want %q", tt.provider, tt.baseURL, got, tt.want)
}
})
}
}
+1 -1
View File
@@ -175,7 +175,7 @@ func (r *QualificationRunner) Run(ctx context.Context) (report RunReport, termin
PulseVersion: version.Version,
PulseBaseURL: r.config.Client.config.BaseURL,
DockerTarget: dockerTargetLabel(r.config.Lab.target),
Model: model, Provider: provider, InferenceRoute: inferenceRouteForProvider(provider), ChallengeNonce: r.config.ChallengeNonce,
Model: model, Provider: provider, InferenceRoute: inferenceRouteForProviderEndpoint(provider, settings.ZaiBaseURL), ChallengeNonce: r.config.ChallengeNonce,
CapturedAt: time.Now().UTC(),
}
return nil
+10
View File
@@ -61,6 +61,16 @@ func TestScoreRunAppliesDollarBudgetOnlyToMeteredAPIRoutes(t *testing.T) {
if strings.Contains(strings.Join(subscription.GateFailures, "\n"), "cost budget") {
t.Fatalf("subscription allowance was treated as metered API spend: %+v", subscription.GateFailures)
}
codingPlan := ScoreRun(ScoringInput{
Manifest: manifest, Provider: "zai", Model: "zai:glm-5.2",
InferenceRoute: "coding_plan_allowance",
})
if codingPlan.Cost.Known || codingPlan.Cost.BudgetApplicable || codingPlan.Cost.BillingBasis != "coding_plan_allowance" {
t.Fatalf("unexpected coding-plan cost semantics: %+v", codingPlan.Cost)
}
if strings.Contains(strings.Join(codingPlan.GateFailures, "\n"), "cost budget") {
t.Fatalf("coding-plan allowance was treated as metered API spend: %+v", codingPlan.GateFailures)
}
metered := ScoreRun(ScoringInput{
Manifest: manifest, Provider: "unpriced-api", Model: "unpriced-api:model",
@@ -151,6 +151,9 @@ class AIRuntimeDocsPolicyTest(unittest.TestCase):
self.assertIn("Pulse retains tool execution and policy enforcement", normalized_content)
self.assertIn("not represented as a zero-dollar API price", normalized_content)
self.assertIn("per-run metered-API budget as not applicable", normalized_content)
self.assertIn("inference_route=coding_plan_allowance", content)
self.assertIn("per-run monetary cost unknown", normalized_content)
self.assertIn("standard Z.ai `/api/paas/` endpoint remains a `metered_api` route", normalized_content)
def test_public_ai_privacy_copy_discloses_outbound_usage_telemetry(self) -> None:
content = read_repo_text("docs/AI.md")