mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
Preserve coding-plan provenance in qualification
This commit is contained in:
@@ -415,6 +415,12 @@ is not represented as a zero-dollar API price. The report keeps its monetary
|
||||
cost unknown and marks the per-run metered-API budget as not applicable; plan
|
||||
limits, provider errors, latency, and any usage the CLI exposes remain visible.
|
||||
|
||||
Z.ai requests sent through a configured `/api/coding/paas/` endpoint are
|
||||
recorded as `inference_route=coding_plan_allowance`. Qualification keeps their
|
||||
per-run monetary cost unknown and the metered-API dollar budget not applicable,
|
||||
while still scoring tokens, latency, provider or plan failures, and model
|
||||
quality. The standard Z.ai `/api/paas/` endpoint remains a `metered_api` route.
|
||||
|
||||
### Models
|
||||
|
||||
Pulse uses model identifiers in the form: `provider:model-name`
|
||||
|
||||
@@ -297,6 +297,9 @@ pricing on `metered_api` remains a fail-closed dollar-budget failure. A
|
||||
monetary cost as unknown with `budget_applicable=false`; it must not invent a
|
||||
zero-dollar price, and latency, provider/plan errors, tool efficiency, and any
|
||||
usage counts exposed by the transport remain scored and reportable.
|
||||
The configured Z.ai `/api/coding/paas/` endpoint is likewise reported as
|
||||
`coding_plan_allowance`, with unknown monetary cost and no per-run metered-API
|
||||
dollar budget; the standard `/api/paas/` endpoint remains `metered_api`.
|
||||
|
||||
## Canonical Files
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ type AISettings struct {
|
||||
PatrolEnabled bool `json:"patrol_enabled"`
|
||||
CodexSubscriptionEnabled bool `json:"codex_subscription_enabled"`
|
||||
ClaudeSubscriptionEnabled bool `json:"claude_subscription_enabled"`
|
||||
ZaiBaseURL string `json:"zai_base_url"`
|
||||
}
|
||||
|
||||
type PulseVersion struct {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
@@ -52,6 +53,18 @@ func inferenceRouteForProvider(provider string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func inferenceRouteForProviderEndpoint(provider, baseURL string) string {
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
if provider != "zai" {
|
||||
return inferenceRouteForProvider(provider)
|
||||
}
|
||||
parsed, err := url.Parse(strings.TrimSpace(baseURL))
|
||||
if err == nil && strings.Contains(strings.ToLower(parsed.Path), "/coding/paas/") {
|
||||
return "coding_plan_allowance"
|
||||
}
|
||||
return inferenceRouteForProvider(provider)
|
||||
}
|
||||
|
||||
type RunReport struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
RunID string `json:"run_id"`
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package qualification
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestInferenceRouteForProviderEndpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
provider string
|
||||
baseURL string
|
||||
want string
|
||||
}{
|
||||
{name: "Zai coding plan", provider: "zai", baseURL: "https://api.z.ai/api/coding/paas/v4", want: "coding_plan_allowance"},
|
||||
{name: "Zai coding completions", provider: "ZAI", baseURL: "https://api.z.ai/api/coding/paas/v4/chat/completions", want: "coding_plan_allowance"},
|
||||
{name: "Zai metered API", provider: "zai", baseURL: "https://api.z.ai/api/paas/v4", want: "metered_api"},
|
||||
{name: "unparseable Zai endpoint", provider: "zai", baseURL: "://", want: "metered_api"},
|
||||
{name: "local subscription agent", provider: "codex-subscription", baseURL: "", want: "local_subscription_agent"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := inferenceRouteForProviderEndpoint(tt.provider, tt.baseURL); got != tt.want {
|
||||
t.Fatalf("inferenceRouteForProviderEndpoint(%q, %q) = %q, want %q", tt.provider, tt.baseURL, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -175,7 +175,7 @@ func (r *QualificationRunner) Run(ctx context.Context) (report RunReport, termin
|
||||
PulseVersion: version.Version,
|
||||
PulseBaseURL: r.config.Client.config.BaseURL,
|
||||
DockerTarget: dockerTargetLabel(r.config.Lab.target),
|
||||
Model: model, Provider: provider, InferenceRoute: inferenceRouteForProvider(provider), ChallengeNonce: r.config.ChallengeNonce,
|
||||
Model: model, Provider: provider, InferenceRoute: inferenceRouteForProviderEndpoint(provider, settings.ZaiBaseURL), ChallengeNonce: r.config.ChallengeNonce,
|
||||
CapturedAt: time.Now().UTC(),
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -61,6 +61,16 @@ func TestScoreRunAppliesDollarBudgetOnlyToMeteredAPIRoutes(t *testing.T) {
|
||||
if strings.Contains(strings.Join(subscription.GateFailures, "\n"), "cost budget") {
|
||||
t.Fatalf("subscription allowance was treated as metered API spend: %+v", subscription.GateFailures)
|
||||
}
|
||||
codingPlan := ScoreRun(ScoringInput{
|
||||
Manifest: manifest, Provider: "zai", Model: "zai:glm-5.2",
|
||||
InferenceRoute: "coding_plan_allowance",
|
||||
})
|
||||
if codingPlan.Cost.Known || codingPlan.Cost.BudgetApplicable || codingPlan.Cost.BillingBasis != "coding_plan_allowance" {
|
||||
t.Fatalf("unexpected coding-plan cost semantics: %+v", codingPlan.Cost)
|
||||
}
|
||||
if strings.Contains(strings.Join(codingPlan.GateFailures, "\n"), "cost budget") {
|
||||
t.Fatalf("coding-plan allowance was treated as metered API spend: %+v", codingPlan.GateFailures)
|
||||
}
|
||||
|
||||
metered := ScoreRun(ScoringInput{
|
||||
Manifest: manifest, Provider: "unpriced-api", Model: "unpriced-api:model",
|
||||
|
||||
@@ -151,6 +151,9 @@ class AIRuntimeDocsPolicyTest(unittest.TestCase):
|
||||
self.assertIn("Pulse retains tool execution and policy enforcement", normalized_content)
|
||||
self.assertIn("not represented as a zero-dollar API price", normalized_content)
|
||||
self.assertIn("per-run metered-API budget as not applicable", normalized_content)
|
||||
self.assertIn("inference_route=coding_plan_allowance", content)
|
||||
self.assertIn("per-run monetary cost unknown", normalized_content)
|
||||
self.assertIn("standard Z.ai `/api/paas/` endpoint remains a `metered_api` route", normalized_content)
|
||||
|
||||
def test_public_ai_privacy_copy_discloses_outbound_usage_telemetry(self) -> None:
|
||||
content = read_repo_text("docs/AI.md")
|
||||
|
||||
Reference in New Issue
Block a user