From a8e42fc7fe37cd091682dd9b04883598bed4b10a Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 15 Jul 2026 00:06:56 +0100 Subject: [PATCH] Preserve coding-plan provenance in qualification --- docs/AI.md | 6 +++++ .../v6/internal/subsystems/ai-runtime.md | 3 +++ internal/ai/qualification/client.go | 1 + internal/ai/qualification/report.go | 13 ++++++++++ .../report_inference_route_test.go | 25 +++++++++++++++++++ internal/ai/qualification/runner.go | 2 +- internal/ai/qualification/scorer_test.go | 10 ++++++++ .../ai_runtime_docs_policy_test.py | 3 +++ 8 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 internal/ai/qualification/report_inference_route_test.go diff --git a/docs/AI.md b/docs/AI.md index d82adc38b..5b9f45174 100644 --- a/docs/AI.md +++ b/docs/AI.md @@ -415,6 +415,12 @@ is not represented as a zero-dollar API price. The report keeps its monetary cost unknown and marks the per-run metered-API budget as not applicable; plan limits, provider errors, latency, and any usage the CLI exposes remain visible. +Z.ai requests sent through a configured `/api/coding/paas/` endpoint are +recorded as `inference_route=coding_plan_allowance`. Qualification keeps their +per-run monetary cost unknown and the metered-API dollar budget not applicable, +while still scoring tokens, latency, provider or plan failures, and model +quality. The standard Z.ai `/api/paas/` endpoint remains a `metered_api` route. + ### Models Pulse uses model identifiers in the form: `provider:model-name` diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index af01586bb..a8dd7892f 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -297,6 +297,9 @@ pricing on `metered_api` remains a fail-closed dollar-budget failure. A monetary cost as unknown with `budget_applicable=false`; it must not invent a zero-dollar price, and latency, provider/plan errors, tool efficiency, and any usage counts exposed by the transport remain scored and reportable. +The configured Z.ai `/api/coding/paas/` endpoint is likewise reported as +`coding_plan_allowance`, with unknown monetary cost and no per-run metered-API +dollar budget; the standard `/api/paas/` endpoint remains `metered_api`. ## Canonical Files diff --git a/internal/ai/qualification/client.go b/internal/ai/qualification/client.go index d8c727e5d..2c38fe888 100644 --- a/internal/ai/qualification/client.go +++ b/internal/ai/qualification/client.go @@ -54,6 +54,7 @@ type AISettings struct { PatrolEnabled bool `json:"patrol_enabled"` CodexSubscriptionEnabled bool `json:"codex_subscription_enabled"` ClaudeSubscriptionEnabled bool `json:"claude_subscription_enabled"` + ZaiBaseURL string `json:"zai_base_url"` } type PulseVersion struct { diff --git a/internal/ai/qualification/report.go b/internal/ai/qualification/report.go index 053173e9f..dfd0a6805 100644 --- a/internal/ai/qualification/report.go +++ b/internal/ai/qualification/report.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "html" + "net/url" "os" "path/filepath" "regexp" @@ -52,6 +53,18 @@ func inferenceRouteForProvider(provider string) string { } } +func inferenceRouteForProviderEndpoint(provider, baseURL string) string { + provider = strings.ToLower(strings.TrimSpace(provider)) + if provider != "zai" { + return inferenceRouteForProvider(provider) + } + parsed, err := url.Parse(strings.TrimSpace(baseURL)) + if err == nil && strings.Contains(strings.ToLower(parsed.Path), "/coding/paas/") { + return "coding_plan_allowance" + } + return inferenceRouteForProvider(provider) +} + type RunReport struct { SchemaVersion string `json:"schema_version"` RunID string `json:"run_id"` diff --git a/internal/ai/qualification/report_inference_route_test.go b/internal/ai/qualification/report_inference_route_test.go new file mode 100644 index 000000000..f19b5fa70 --- /dev/null +++ b/internal/ai/qualification/report_inference_route_test.go @@ -0,0 +1,25 @@ +package qualification + +import "testing" + +func TestInferenceRouteForProviderEndpoint(t *testing.T) { + tests := []struct { + name string + provider string + baseURL string + want string + }{ + {name: "Zai coding plan", provider: "zai", baseURL: "https://api.z.ai/api/coding/paas/v4", want: "coding_plan_allowance"}, + {name: "Zai coding completions", provider: "ZAI", baseURL: "https://api.z.ai/api/coding/paas/v4/chat/completions", want: "coding_plan_allowance"}, + {name: "Zai metered API", provider: "zai", baseURL: "https://api.z.ai/api/paas/v4", want: "metered_api"}, + {name: "unparseable Zai endpoint", provider: "zai", baseURL: "://", want: "metered_api"}, + {name: "local subscription agent", provider: "codex-subscription", baseURL: "", want: "local_subscription_agent"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := inferenceRouteForProviderEndpoint(tt.provider, tt.baseURL); got != tt.want { + t.Fatalf("inferenceRouteForProviderEndpoint(%q, %q) = %q, want %q", tt.provider, tt.baseURL, got, tt.want) + } + }) + } +} diff --git a/internal/ai/qualification/runner.go b/internal/ai/qualification/runner.go index a271bca7d..4f3956b83 100644 --- a/internal/ai/qualification/runner.go +++ b/internal/ai/qualification/runner.go @@ -175,7 +175,7 @@ func (r *QualificationRunner) Run(ctx context.Context) (report RunReport, termin PulseVersion: version.Version, PulseBaseURL: r.config.Client.config.BaseURL, DockerTarget: dockerTargetLabel(r.config.Lab.target), - Model: model, Provider: provider, InferenceRoute: inferenceRouteForProvider(provider), ChallengeNonce: r.config.ChallengeNonce, + Model: model, Provider: provider, InferenceRoute: inferenceRouteForProviderEndpoint(provider, settings.ZaiBaseURL), ChallengeNonce: r.config.ChallengeNonce, CapturedAt: time.Now().UTC(), } return nil diff --git a/internal/ai/qualification/scorer_test.go b/internal/ai/qualification/scorer_test.go index 5695ea93c..b52260f82 100644 --- a/internal/ai/qualification/scorer_test.go +++ b/internal/ai/qualification/scorer_test.go @@ -61,6 +61,16 @@ func TestScoreRunAppliesDollarBudgetOnlyToMeteredAPIRoutes(t *testing.T) { if strings.Contains(strings.Join(subscription.GateFailures, "\n"), "cost budget") { t.Fatalf("subscription allowance was treated as metered API spend: %+v", subscription.GateFailures) } + codingPlan := ScoreRun(ScoringInput{ + Manifest: manifest, Provider: "zai", Model: "zai:glm-5.2", + InferenceRoute: "coding_plan_allowance", + }) + if codingPlan.Cost.Known || codingPlan.Cost.BudgetApplicable || codingPlan.Cost.BillingBasis != "coding_plan_allowance" { + t.Fatalf("unexpected coding-plan cost semantics: %+v", codingPlan.Cost) + } + if strings.Contains(strings.Join(codingPlan.GateFailures, "\n"), "cost budget") { + t.Fatalf("coding-plan allowance was treated as metered API spend: %+v", codingPlan.GateFailures) + } metered := ScoreRun(ScoringInput{ Manifest: manifest, Provider: "unpriced-api", Model: "unpriced-api:model", diff --git a/scripts/release_control/ai_runtime_docs_policy_test.py b/scripts/release_control/ai_runtime_docs_policy_test.py index 3d41be31d..31ef19a69 100644 --- a/scripts/release_control/ai_runtime_docs_policy_test.py +++ b/scripts/release_control/ai_runtime_docs_policy_test.py @@ -151,6 +151,9 @@ class AIRuntimeDocsPolicyTest(unittest.TestCase): self.assertIn("Pulse retains tool execution and policy enforcement", normalized_content) self.assertIn("not represented as a zero-dollar API price", normalized_content) self.assertIn("per-run metered-API budget as not applicable", normalized_content) + self.assertIn("inference_route=coding_plan_allowance", content) + self.assertIn("per-run monetary cost unknown", normalized_content) + self.assertIn("standard Z.ai `/api/paas/` endpoint remains a `metered_api` route", normalized_content) def test_public_ai_privacy_copy_discloses_outbound_usage_telemetry(self) -> None: content = read_repo_text("docs/AI.md")