From abb817c7d54158f5db7f221cdb2ee751efb4760f Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Thu, 27 Aug 2026 21:13:44 +1000 Subject: [PATCH] feat: add admin Ochre catalog read and drop provider-direct model (#917) Expose per-model availability and reason on an operator-only ListOchreCatalog action for the console, keeping the tenant ListFoundationModels omission intact. Remove the unbuilt provider-direct entry from the v1 catalog. --- cmd/spinifex/cmd/admin_ochre.go | 21 +-- cmd/spinifex/cmd/admin_ochre_pull_test.go | 10 +- cmd/spinifex/cmd/admin_ochre_test.go | 30 ++--- spinifex/gateway/bedrock/access_test.go | 7 +- spinifex/gateway/bedrock/admin_catalog.go | 123 ++++++++++++++++++ .../gateway/bedrock/admin_catalog_test.go | 119 +++++++++++++++++ spinifex/gateway/bedrock/catalog.go | 19 +-- spinifex/gateway/bedrock/catalog_test.go | 6 + .../bedrock/guardrail_inference_test.go | 1 + spinifex/gateway/bedrock/invoke_test.go | 26 ++++ spinifex/gateway/bedrock/prices_test.go | 9 +- spinifex/gateway/bedrock/provisioned_test.go | 1 + spinifex/gateway/bedrock/router_test.go | 3 + spinifex/gateway/spinifex.go | 6 + tests/e2e/bedrock/bedrock_test.go | 42 +++--- 15 files changed, 350 insertions(+), 73 deletions(-) create mode 100644 spinifex/gateway/bedrock/admin_catalog.go create mode 100644 spinifex/gateway/bedrock/admin_catalog_test.go diff --git a/cmd/spinifex/cmd/admin_ochre.go b/cmd/spinifex/cmd/admin_ochre.go index 8c70fa545..d1de126d1 100644 --- a/cmd/spinifex/cmd/admin_ochre.go +++ b/cmd/spinifex/cmd/admin_ochre.go @@ -351,6 +351,17 @@ func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore. return uploaded, nil } +// weightsStagingRefusal builds the model-ID refusal error weights staging and +// pulling share. Unknown IDs and provider-served IDs get distinct wording so +// a caller can tell which one it hit; taking found/selfHost as plain bools +// keeps this testable without a real catalog entry for the provider case. +func weightsStagingRefusal(modelID, operation string, found, _ bool) error { + if !found { + return fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID) + } + return fmt.Errorf("%q is a provider-served model, not self-host; weights %s does not apply", modelID, operation) +} + // runPullWeights holds 'ochre weights pull' decision and side-effect logic: // catalog validation, ref resolution to an immutable commit SHA (D3), tree // listing and safetensors-only filtering (D4), streaming each selected file @@ -360,10 +371,7 @@ func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore. func runPullWeights(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, modelID, hfRepo, revision string, revisionExplicit bool, s3URIFlag string) (string, error) { spec, found, selfHost := gateway_bedrock.LookupServingSpec(modelID) if !found || !selfHost { - if !found { - return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID) - } - return "", fmt.Errorf("%q is a provider-served model, not self-host; weights pull does not apply", modelID) + return "", weightsStagingRefusal(modelID, "pull", found, selfHost) } // The catalog carries the canonical repo and a pinned revision so a bare @@ -667,10 +675,7 @@ type weightsSnapshotChecker func(ctx context.Context, snapshotID string) (bool, // fake materializer. func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weightsStore *gateway_bedrock.WeightsStore, tmpDirFlag, modelID, s3URI string, materialize weightsMaterializer, checkSnapshotLive weightsSnapshotChecker) (string, error) { if _, found, selfHost := gateway_bedrock.LookupServingSpec(modelID); !found || !selfHost { - if !found { - return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID) - } - return "", fmt.Errorf("%q is a provider-served model, not self-host; weights staging does not apply", modelID) + return "", weightsStagingRefusal(modelID, "staging", found, selfHost) } bucket, prefix, err := parseWeightsS3URI(s3URI) diff --git a/cmd/spinifex/cmd/admin_ochre_pull_test.go b/cmd/spinifex/cmd/admin_ochre_pull_test.go index 4f7e81dd5..947644658 100644 --- a/cmd/spinifex/cmd/admin_ochre_pull_test.go +++ b/cmd/spinifex/cmd/admin_ochre_pull_test.go @@ -235,10 +235,12 @@ func TestRunPullWeights_UnknownModelRefused(t *testing.T) { assert.Contains(t, err.Error(), "not present in the Ochre catalog") } -// TestRunPullWeights_ProviderModelRefused covers the self-host gate: a -// provider-served model must refuse before any network work, same as stage. -func TestRunPullWeights_ProviderModelRefused(t *testing.T) { - _, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, providerModelID, testHFRepo, "main", true, "") +// TestWeightsStagingRefusal_PullOperationWording covers the self-host gate's +// message for the 'pull' operation. v1 ships no provider-tier catalog entry, +// so this drives weightsStagingRefusal directly rather than through a real +// LookupServingSpec lookup. +func TestWeightsStagingRefusal_PullOperationWording(t *testing.T) { + err := weightsStagingRefusal(providerModelID, "pull", true, false) require.Error(t, err) assert.Contains(t, err.Error(), "not self-host") } diff --git a/cmd/spinifex/cmd/admin_ochre_test.go b/cmd/spinifex/cmd/admin_ochre_test.go index 1c0fec5b8..f1ef4b818 100644 --- a/cmd/spinifex/cmd/admin_ochre_test.go +++ b/cmd/spinifex/cmd/admin_ochre_test.go @@ -25,10 +25,11 @@ import ( "github.com/stretchr/testify/require" ) -// selfHostModelID and providerModelID are real catalog entries: the former -// is self-host (weights staging applies), the latter is provider-served -// (weights staging must refuse it), so tests exercise LookupServingSpec's -// found/selfHost distinction against the real catalog rather than a stub. +// selfHostModelID is a real catalog entry (weights staging applies). +// providerModelID is an arbitrary ID not present in the catalog at all: v1 +// ships no provider-tier entry, so the found-but-not-selfHost case is +// exercised directly against weightsStagingRefusal instead, not through a +// real LookupServingSpec lookup. const ( selfHostModelID = "meta.llama3-2-1b-instruct-v1:0" providerModelID = "anthropic.claude-3-5-sonnet-20240620-v1:0" @@ -404,24 +405,17 @@ func TestRunStageWeights_UnknownModelIDRefused(t *testing.T) { assert.Empty(t, entries, "refusal must not write a KV entry") } -// TestRunStageWeights_ProviderModelIDRefused covers refusal on a model ID -// that exists in the catalog but is provider-served, not self-host. The -// error must be distinguishable from the unknown-model-ID case, since -// LookupServingSpec returns found and selfHost separately for exactly this -// reason. -func TestRunStageWeights_ProviderModelIDRefused(t *testing.T) { - weightsStore := newWeightsStoreForTest(t) - - _, err := runStageWeights(context.Background(), explodingObjectStore{t: t}, weightsStore, t.TempDir(), - providerModelID, "s3://models/claude/", explodingMaterializer(t), explodingSnapshotChecker(t)) +// TestWeightsStagingRefusal_ProviderServedDistinguishesFromUnknown covers +// refusal for a model ID that is found in the catalog but is provider-served, +// not self-host. The error must be distinguishable from the unknown-model-ID +// case; v1 ships no provider-tier entry, so this drives weightsStagingRefusal +// directly with explicit found/selfHost values rather than a real lookup. +func TestWeightsStagingRefusal_ProviderServedDistinguishesFromUnknown(t *testing.T) { + err := weightsStagingRefusal(providerModelID, "staging", true, false) require.Error(t, err) assert.ErrorContains(t, err, "provider-served") assert.NotContains(t, err.Error(), "unknown model ID", "must be distinguishable from the unknown-model-ID refusal") - - entries, listErr := weightsStore.ListWeights(context.Background()) - require.NoError(t, listErr) - assert.Empty(t, entries, "refusal must not write a KV entry") } // TestRunStageWeights_MissingRequiredFilesRefusedBeforeMaterializing covers diff --git a/spinifex/gateway/bedrock/access_test.go b/spinifex/gateway/bedrock/access_test.go index f65e8a880..5e3a5a312 100644 --- a/spinifex/gateway/bedrock/access_test.go +++ b/spinifex/gateway/bedrock/access_test.go @@ -12,9 +12,10 @@ import ( "github.com/stretchr/testify/require" ) -// Shipped catalog entries, one per tier, referenced by the access tests. -// Naming them here keeps a catalog change to one edit per tier rather than one -// per assertion. +// selfHostTestModel and selfHostTestModel3B are shipped catalog entries. +// anthropicTestModel is not: v1 ships no provider-tier entry, so +// provider-path tests pair it with withProviderCatalogEntry. Naming them here +// keeps a catalog change to one edit per tier rather than one per assertion. const ( selfHostTestModel = "meta.llama3-2-1b-instruct-v1:0" selfHostTestModel3B = "meta.llama3-2-3b-instruct-v1:0" diff --git a/spinifex/gateway/bedrock/admin_catalog.go b/spinifex/gateway/bedrock/admin_catalog.go new file mode 100644 index 000000000..14834d93d --- /dev/null +++ b/spinifex/gateway/bedrock/admin_catalog.go @@ -0,0 +1,123 @@ +package gateway_bedrock + +import ( + "context" + "fmt" + "log/slog" + "strings" +) + +// Availability values an AdminCatalogEntry carries. AvailabilityAvailable +// means the model is servable to accountID today; the other three are the +// same reasons tieredCatalog computes and collapses to omission on the +// tenant-facing ListFoundationModels path. +const ( + AvailabilityAvailable = "available" + AvailabilityUngranted = "ungranted" + AvailabilityNoWeights = "no-weights-staged" + // #nosec G101 -- an availability-reason label, not a credential value. + AvailabilityNoCredential = "no-credential" +) + +// AdminCatalogEntry is one catalog model as an operator sees it: every field +// ListFoundationModels exposes plus the ops metadata AWS's wire shape cannot +// carry (VRAM floor, instance type, co-serve group) and the specific +// availability reason tieredCatalog would otherwise collapse to omission. +type AdminCatalogEntry struct { + ModelID string `json:"modelId"` + ModelName string `json:"modelName"` + Family string `json:"family"` + InputModalities []string `json:"inputModalities"` + OutputModalities []string `json:"outputModalities"` + ResponseStreamingSupported bool `json:"responseStreamingSupported"` + InputPriceMicroUSDPerMillion int64 `json:"inputPriceMicroUsdPerMillion"` + OutputPriceMicroUSDPerMillion int64 `json:"outputPriceMicroUsdPerMillion"` + PriceKnown bool `json:"priceKnown"` + MinVRAMMiB int `json:"minVramMib"` + InstanceType string `json:"instanceType"` + CoServeGroup string `json:"coServeGroup"` + Availability string `json:"availability"` +} + +// AdminCatalogList is the ListOchreCatalog action's response envelope. +type AdminCatalogList struct { + Entries []AdminCatalogEntry `json:"entries"` +} + +// toAdminEntry projects e into its admin-read shape under the given +// availability reason. +func (e catalogEntry) toAdminEntry(availability string) AdminCatalogEntry { + return AdminCatalogEntry{ + ModelID: e.ModelID, + ModelName: e.ModelName, + Family: e.Family, + InputModalities: e.InputModalities, + OutputModalities: e.OutputModalities, + ResponseStreamingSupported: e.ResponseStreamingSupported, + InputPriceMicroUSDPerMillion: e.InputPriceMicroUSDPerMillion, + OutputPriceMicroUSDPerMillion: e.OutputPriceMicroUSDPerMillion, + PriceKnown: e.PriceKnown, + MinVRAMMiB: e.MinVRAMMiB, + InstanceType: e.InstanceType, + CoServeGroup: e.CoServeGroup, + Availability: availability, + } +} + +// availabilityReason runs the same three checks tieredCatalog uses to decide +// whether to include entry for accountID, but returns which one failed +// instead of a bool: an operator-only view is allowed to say why, where the +// tenant-facing catalog is not. +func availabilityReason(ctx context.Context, accountID string, entry catalogEntry, resolver CredentialResolver, access AccessResolver) (string, error) { + granted, err := access.Granted(ctx, accountID, entry.ModelID) + if err != nil { + return "", err + } + if !granted { + return AvailabilityUngranted, nil + } + + if entry.Provider == tierSelfHost { + _, resolvable, err := currentWeightsResolver().Resolve(ctx, entry.ModelID) + if err != nil { + slog.Error("bedrock: weights resolve failed", "model", entry.ModelID, "err", err) + return "", fmt.Errorf("resolve weights for %s: %w", entry.ModelID, err) + } + if !resolvable { + return AvailabilityNoWeights, nil + } + return AvailabilityAvailable, nil + } + + vendor, ok := strings.CutPrefix(entry.Provider, providerPrefix) + if !ok { + return AvailabilityNoCredential, nil + } + _, resolvable, err := resolver.Resolve(ctx, accountID, vendor) + if err != nil { + return "", err + } + if !resolvable { + return AvailabilityNoCredential, nil + } + return AvailabilityAvailable, nil +} + +// AdminCatalog returns every catalog entry, unfiltered, with its +// availability computed for accountID: the honest operator read Open +// Decision #1 calls for, as opposed to ListFoundationModels which omits +// what accountID cannot use rather than say why. Callers must scope this to +// an operator-only surface — the reason it returns is exactly what +// grantedCatalogEntry and GetFoundationModel deliberately withhold from a +// tenant to avoid confirming a model's existence to a probing account. +func AdminCatalog(ctx context.Context, accountID string, resolver CredentialResolver, access AccessResolver) ([]AdminCatalogEntry, error) { + out := make([]AdminCatalogEntry, 0, len(catalog)) + for _, entry := range catalog { + reason, err := availabilityReason(ctx, accountID, entry, resolver, access) + if err != nil { + return nil, err + } + out = append(out, entry.toAdminEntry(reason)) + } + return out, nil +} diff --git a/spinifex/gateway/bedrock/admin_catalog_test.go b/spinifex/gateway/bedrock/admin_catalog_test.go new file mode 100644 index 000000000..19304e153 --- /dev/null +++ b/spinifex/gateway/bedrock/admin_catalog_test.go @@ -0,0 +1,119 @@ +package gateway_bedrock + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/service/bedrock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func adminEntry(t *testing.T, entries []AdminCatalogEntry, modelID string) AdminCatalogEntry { + t.Helper() + for _, e := range entries { + if e.ModelID == modelID { + return e + } + } + require.Failf(t, "model not found in admin catalog", "modelID=%q", modelID) + return AdminCatalogEntry{} +} + +// TestAdminCatalog_ClaudeEntryAbsent covers the v1 removal directly: the +// shipped catalog carries no Anthropic entry at all, so it cannot appear in +// the admin read even for the operator account. +func TestAdminCatalog_ClaudeEntryAbsent(t *testing.T) { + withWeightsResolver(t, stubWeightsResolver{ok: map[string]bool{}}) + entries, err := AdminCatalog(context.Background(), "000000000000", stubResolver{ok: map[string]bool{}}, grantAll{}) + require.NoError(t, err) + for _, e := range entries { + assert.NotEqual(t, anthropicTestModel, e.ModelID) + } +} + +// TestAdminCatalog_ReasonUngranted covers the ungranted case: the grant check +// runs before any weights/credential resolution, so an ungranted account +// never learns whether the model would otherwise be servable. +func TestAdminCatalog_ReasonUngranted(t *testing.T) { + withWeightsResolver(t, stubWeightsResolver{ok: map[string]bool{selfHostTestModel: true}}) + entries, err := AdminCatalog(context.Background(), "000000000001", stubResolver{ok: map[string]bool{}}, grantSet{}) + require.NoError(t, err) + assert.Equal(t, AvailabilityUngranted, adminEntry(t, entries, selfHostTestModel).Availability) +} + +// TestAdminCatalog_ReasonNoWeightsStaged covers a granted self-host model +// whose weights snapshot does not resolve. +func TestAdminCatalog_ReasonNoWeightsStaged(t *testing.T) { + withWeightsResolver(t, stubWeightsResolver{ok: map[string]bool{}}) + entries, err := AdminCatalog(context.Background(), "000000000001", stubResolver{ok: map[string]bool{}}, grantAll{}) + require.NoError(t, err) + assert.Equal(t, AvailabilityNoWeights, adminEntry(t, entries, selfHostTestModel).Availability) +} + +// TestAdminCatalog_ReasonNoCredential covers a granted provider model whose +// vendor credential does not resolve. v1 ships no provider entry, so this +// synthesizes one the same way the router tests do. +func TestAdminCatalog_ReasonNoCredential(t *testing.T) { + withProviderCatalogEntry(t, anthropicTestModel) + withWeightsResolver(t, stubWeightsResolver{ok: map[string]bool{selfHostTestModel: true}}) + entries, err := AdminCatalog(context.Background(), "000000000001", stubResolver{ok: map[string]bool{}}, grantAll{}) + require.NoError(t, err) + assert.Equal(t, AvailabilityNoCredential, adminEntry(t, entries, anthropicTestModel).Availability) +} + +// TestAdminCatalog_ReasonAvailable covers the fully-servable case for both +// tiers: granted, and weights/credential resolve. +func TestAdminCatalog_ReasonAvailable(t *testing.T) { + withProviderCatalogEntry(t, anthropicTestModel) + withWeightsResolver(t, stubWeightsResolver{ok: map[string]bool{selfHostTestModel: true}}) + entries, err := AdminCatalog(context.Background(), "000000000001", stubResolver{ok: map[string]bool{"anthropic": true}}, grantAll{}) + require.NoError(t, err) + assert.Equal(t, AvailabilityAvailable, adminEntry(t, entries, selfHostTestModel).Availability) + assert.Equal(t, AvailabilityAvailable, adminEntry(t, entries, anthropicTestModel).Availability) +} + +// TestAdminCatalog_CarriesOpsMetadata covers the fields AWS's own wire shape +// cannot carry: VRAM floor, instance type and co-serve group must survive +// the admin projection unchanged from the catalog entry. +func TestAdminCatalog_CarriesOpsMetadata(t *testing.T) { + withWeightsResolver(t, stubWeightsResolver{ok: map[string]bool{selfHostTestModel: true}}) + entries, err := AdminCatalog(context.Background(), "000000000001", stubResolver{ok: map[string]bool{}}, grantAll{}) + require.NoError(t, err) + entry := adminEntry(t, entries, selfHostTestModel) + assert.Positive(t, entry.MinVRAMMiB) + assert.NotEmpty(t, entry.InstanceType) + assert.Equal(t, coServeGroupOchreDemo, entry.CoServeGroup) +} + +// TestListFoundationModels_NeverLeaksAvailabilityReason is the tenant-safety +// guard: the wire JSON for the public catalog read must never carry an +// availability/reason field, regardless of why a model was omitted, since a +// tenant credential must not learn why (or whether) a model exists. +func TestListFoundationModels_NeverLeaksAvailabilityReason(t *testing.T) { + withWeightsResolver(t, stubWeightsResolver{ok: map[string]bool{}}) + out, err := ListFoundationModels(context.Background(), "000000000001", stubResolver{ok: map[string]bool{}}, + grantSet{}, &bedrock.ListFoundationModelsInput{}) + require.NoError(t, err) + assert.Empty(t, modelIDs(out), "an ungranted, unresolvable account must see an empty catalog") + + body, err := json.Marshal(out) + require.NoError(t, err) + lower := strings.ToLower(string(body)) + assert.NotContains(t, lower, "availab") + assert.NotContains(t, lower, "reason") + assert.NotContains(t, lower, "ungranted") + assert.NotContains(t, lower, "credential") +} + +// TestListFoundationModels_ClaudeEntryAbsent pins the tenant-facing catalog +// to the same v1 removal AdminCatalog observes. +func TestListFoundationModels_ClaudeEntryAbsent(t *testing.T) { + withWeightsResolver(t, stubWeightsResolver{ok: map[string]bool{selfHostTestModel: true, selfHostTestModel3B: true}}) + out, err := ListFoundationModels(context.Background(), "000000000001", stubResolver{ok: map[string]bool{"anthropic": true}}, + grantAll{}, &bedrock.ListFoundationModelsInput{}) + require.NoError(t, err) + assert.NotContains(t, modelIDs(out), anthropicTestModel) +} diff --git a/spinifex/gateway/bedrock/catalog.go b/spinifex/gateway/bedrock/catalog.go index e27d04600..3bc785bd1 100644 --- a/spinifex/gateway/bedrock/catalog.go +++ b/spinifex/gateway/bedrock/catalog.go @@ -79,8 +79,9 @@ type catalogEntry struct { PriceKnown bool // provider entries only; self-host is always known-zero } -// catalog is the static model set: two self-hosted open models and one -// Anthropic-direct model. Later phases extend this list. +// catalog is the static model set: self-hosted open models only. v1 does not +// ship a provider-direct tier; Provider's "provider:" plumbing stays +// in place, unused, for a later phase to re-enable. var catalog = []catalogEntry{ { ModelID: "meta.llama3-2-1b-instruct-v1:0", @@ -178,20 +179,6 @@ var catalog = []catalogEntry{ MinVRAMMiB: 1200, InstanceType: "g5.xlarge", }, - { - ModelID: "anthropic.claude-3-5-sonnet-20240620-v1:0", - ModelName: "Claude 3.5 Sonnet", - ProviderName: "Anthropic", - Provider: providerPrefix + vendorAnthropic, - InputModalities: []string{"TEXT", "IMAGE"}, - OutputModalities: []string{"TEXT"}, - ResponseStreamingSupported: false, - InferenceTypesSupported: []string{"ON_DEMAND"}, - // List pricing at launch: $3/MTok input, $15/MTok output. - InputPriceMicroUSDPerMillion: 3_000_000, - OutputPriceMicroUSDPerMillion: 15_000_000, - PriceKnown: true, - }, } // CatalogModelIDs returns every model ID the platform knows about, ignoring diff --git a/spinifex/gateway/bedrock/catalog_test.go b/spinifex/gateway/bedrock/catalog_test.go index ad45c96f7..b7456d42c 100644 --- a/spinifex/gateway/bedrock/catalog_test.go +++ b/spinifex/gateway/bedrock/catalog_test.go @@ -101,6 +101,7 @@ func TestListFoundationModels_SelfHostExcludedWhenUngranted(t *testing.T) { } func TestListFoundationModels_ProviderIncludedWhenGrantedAndResolvable(t *testing.T) { + withProviderCatalogEntry(t, anthropicTestModel) out, err := ListFoundationModels(context.Background(), "000000000001", stubResolver{ok: map[string]bool{"anthropic": true}}, grantSet{anthropicTestModel: true}, &bedrock.ListFoundationModelsInput{}) require.NoError(t, err) @@ -108,6 +109,7 @@ func TestListFoundationModels_ProviderIncludedWhenGrantedAndResolvable(t *testin } func TestListFoundationModels_ProviderExcludedWhenUnresolvable(t *testing.T) { + withProviderCatalogEntry(t, anthropicTestModel) withWeightsResolver(t, stubWeightsResolver{ok: map[string]bool{}}) out, err := ListFoundationModels(context.Background(), "000000000001", stubResolver{ok: map[string]bool{}}, grantSet{anthropicTestModel: true}, &bedrock.ListFoundationModelsInput{}) @@ -134,6 +136,7 @@ func TestGetFoundationModel_SelfHostWithUnresolvableWeightsReturnsNotFound(t *te // filters independent: a grant says the account may use the model, not that // the platform can reach it. func TestListFoundationModels_ProviderExcludedWhenUngrantedButResolvable(t *testing.T) { + withProviderCatalogEntry(t, anthropicTestModel) out, err := ListFoundationModels(context.Background(), "000000000001", stubResolver{ok: map[string]bool{"anthropic": true}}, grantSet{}, &bedrock.ListFoundationModelsInput{}) require.NoError(t, err) @@ -145,6 +148,7 @@ func TestListFoundationModels_ProviderExcludedWhenUngrantedButResolvable(t *test // with the catalog: a model missing here would be ungrantable via --all-models // and so invisible to every account. func TestCatalogModelIDs_CoversWholeCatalog(t *testing.T) { + withProviderCatalogEntry(t, anthropicTestModel) ids := CatalogModelIDs() require.Len(t, ids, len(catalog)) assert.Contains(t, ids, selfHostTestModel) @@ -226,6 +230,7 @@ func TestLookupServingSpec_EmbedderCarriesPinnedRevision(t *testing.T) { // hit for a valid-but-wrong-tier model ID: found=true (it IS a catalog // entry), selfHost=false, so staging must be refused rather than proceed. func TestLookupServingSpec_ProviderEntry(t *testing.T) { + withProviderCatalogEntry(t, "anthropic.claude-3-5-sonnet-20240620-v1:0") spec, found, selfHost := LookupServingSpec("anthropic.claude-3-5-sonnet-20240620-v1:0") require.True(t, found) assert.False(t, selfHost) @@ -359,6 +364,7 @@ func TestLookupCoServeGroup_Bundle(t *testing.T) { // TestLookupCoServeGroup_ProviderEntry covers the refusal case a launcher // must hit for a valid-but-wrong-tier model ID. func TestLookupCoServeGroup_ProviderEntry(t *testing.T) { + withProviderCatalogEntry(t, anthropicTestModel) spec, found, selfHost := LookupCoServeGroup(anthropicTestModel) require.True(t, found) assert.False(t, selfHost) diff --git a/spinifex/gateway/bedrock/guardrail_inference_test.go b/spinifex/gateway/bedrock/guardrail_inference_test.go index 2eb884a1d..943e52178 100644 --- a/spinifex/gateway/bedrock/guardrail_inference_test.go +++ b/spinifex/gateway/bedrock/guardrail_inference_test.go @@ -432,6 +432,7 @@ func TestInvokeRouter_GuardrailInputBlock_Anthropic_SkipsBackend(t *testing.T) { require.NoError(t, err) modelID := "anthropic.claude-3-5-sonnet-20240620-v1:0" + withProviderCatalogEntry(t, modelID) rt := NewInvokeRouter(stubCredentialResolver{key: "sk-test", ok: true}, nil, nil, grantAll{}, nil, store, nil) body := []byte(`{"anthropic_version":"bedrock-2023-05-31","max_tokens":100,"messages":[{"role":"user","content":[{"type":"text","text":"this has a badword in it"}]}]}`) diff --git a/spinifex/gateway/bedrock/invoke_test.go b/spinifex/gateway/bedrock/invoke_test.go index eb06d0025..050fda5c8 100644 --- a/spinifex/gateway/bedrock/invoke_test.go +++ b/spinifex/gateway/bedrock/invoke_test.go @@ -39,6 +39,30 @@ func withCatalogEntry(t *testing.T, entry catalogEntry) { t.Cleanup(func() { catalog = original }) } +// withProviderCatalogEntry installs a synthetic provider-tier entry for +// modelID under the Anthropic vendor, the only one Router/InvokeRouter +// dispatch to. v1 ships no provider-tier catalog entry of its own, so +// provider-path tests (credential gating, dispatch, in-tree pricing) +// synthesize one instead of depending on a real catalog member. +func withProviderCatalogEntry(t *testing.T, modelID string) catalogEntry { + t.Helper() + entry := catalogEntry{ + ModelID: modelID, + ModelName: "Test Anthropic Model", + ProviderName: "Anthropic", + Provider: providerPrefix + vendorAnthropic, + InputModalities: []string{"TEXT", "IMAGE"}, + OutputModalities: []string{"TEXT"}, + ResponseStreamingSupported: true, + InferenceTypesSupported: []string{"ON_DEMAND"}, + InputPriceMicroUSDPerMillion: 3_000_000, + OutputPriceMicroUSDPerMillion: 15_000_000, + PriceKnown: true, + } + withCatalogEntry(t, entry) + return entry +} + func TestInvokeRouter_SelfHostSuccess(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -70,6 +94,7 @@ func TestInvokeRouter_UnknownModelReturnsResourceNotFound(t *testing.T) { } func TestInvokeRouter_AnthropicNoCredentialReturnsAccessDenied(t *testing.T) { + withProviderCatalogEntry(t, "anthropic.claude-3-5-sonnet-20240620-v1:0") rt := NewInvokeRouter(stubCredentialResolver{ok: false}, nil, nil, grantAll{}, nil, nil, nil) _, _, err := rt.InvokeModel(context.Background(), "000000000001", "anthropic.claude-3-5-sonnet-20240620-v1:0", []byte(`{}`), "", "") require.Error(t, err) @@ -137,6 +162,7 @@ func TestInvokeStreamRouter_UnknownModelReturnsResourceNotFound(t *testing.T) { } func TestInvokeStreamRouter_AnthropicNoCredentialReturnsAccessDenied(t *testing.T) { + withProviderCatalogEntry(t, "anthropic.claude-3-5-sonnet-20240620-v1:0") rt := NewInvokeStreamRouter(stubCredentialResolver{ok: false}, nil, grantAll{}, nil, nil, nil) _, err := rt.InvokeModelWithResponseStream(context.Background(), "000000000001", "anthropic.claude-3-5-sonnet-20240620-v1:0", []byte(`{}`), "", "") require.Error(t, err) diff --git a/spinifex/gateway/bedrock/prices_test.go b/spinifex/gateway/bedrock/prices_test.go index 573a7f059..4105583aa 100644 --- a/spinifex/gateway/bedrock/prices_test.go +++ b/spinifex/gateway/bedrock/prices_test.go @@ -47,8 +47,7 @@ func TestResolvePrice_SelfHost_AlwaysKnownZero(t *testing.T) { // TestResolvePrice_ProviderUsesInTreeDefault covers a provider entry with no // KV override: it falls back to the catalog's in-tree default price. func TestResolvePrice_ProviderUsesInTreeDefault(t *testing.T) { - entry, ok := lookupCatalogEntry(priceTestProviderModelID) - require.True(t, ok) + entry := withProviderCatalogEntry(t, priceTestProviderModelID) price, err := resolvePrice(context.Background(), nil, entry) require.NoError(t, err) @@ -61,8 +60,7 @@ func TestResolvePrice_ProviderUsesInTreeDefault(t *testing.T) { // with a KV override present: the override wins outright, not just fills a // gap in the in-tree default. func TestResolvePrice_KVOverrideWinsOverInTreeDefault(t *testing.T) { - entry, ok := lookupCatalogEntry(priceTestProviderModelID) - require.True(t, ok) + entry := withProviderCatalogEntry(t, priceTestProviderModelID) override := Price{InputMicroUSDPerMillion: 1_000_000, OutputMicroUSDPerMillion: 2_000_000, Known: true} price, err := resolvePrice(context.Background(), stubPriceResolver{price: override, ok: true}, entry) @@ -86,8 +84,7 @@ func TestResolvePrice_NoDefaultNoOverride_Unknown(t *testing.T) { // surfaces rather than silently falling through to the in-tree default or an // unknown price — an internal fault is not a pricing verdict. func TestResolvePrice_ResolverErrorPropagates(t *testing.T) { - entry, ok := lookupCatalogEntry(priceTestProviderModelID) - require.True(t, ok) + entry := withProviderCatalogEntry(t, priceTestProviderModelID) _, err := resolvePrice(context.Background(), stubPriceResolver{err: assert.AnError}, entry) require.Error(t, err) diff --git a/spinifex/gateway/bedrock/provisioned_test.go b/spinifex/gateway/bedrock/provisioned_test.go index e4f9234ca..236177e5b 100644 --- a/spinifex/gateway/bedrock/provisioned_test.go +++ b/spinifex/gateway/bedrock/provisioned_test.go @@ -118,6 +118,7 @@ func createInput(modelID, name string, units int64) *bedrock.CreateProvisionedMo // bead's core constraint: provisioning capacity only makes sense for a model // this platform actually launches VMs for. func TestCreateProvisionedModelThroughput_RejectsProviderHostedModel(t *testing.T) { + withProviderCatalogEntry(t, anthropicTestModel) stub := newStubEndpointProvisioner() store := newProvisionedTestStore(t, stub) diff --git a/spinifex/gateway/bedrock/router_test.go b/spinifex/gateway/bedrock/router_test.go index 285e2f9cd..5fe1dac1a 100644 --- a/spinifex/gateway/bedrock/router_test.go +++ b/spinifex/gateway/bedrock/router_test.go @@ -63,6 +63,7 @@ func TestRouter_Converse_UnknownModelReturnsResourceNotFound(t *testing.T) { } func TestRouter_Converse_AnthropicNoCredentialReturnsAccessDenied(t *testing.T) { + withProviderCatalogEntry(t, "anthropic.claude-3-5-sonnet-20240620-v1:0") rt := NewRouter(stubCredentialResolver{ok: false}, nil, nil, grantAll{}, nil, nil, nil) _, err := rt.Converse(context.Background(), "000000000001", "anthropic.claude-3-5-sonnet-20240620-v1:0", converseInput()) require.Error(t, err) @@ -128,6 +129,7 @@ func TestRouter_ConverseStream_UnknownModelReturnsResourceNotFound(t *testing.T) } func TestRouter_ConverseStream_AnthropicNoCredentialReturnsAccessDenied(t *testing.T) { + withProviderCatalogEntry(t, "anthropic.claude-3-5-sonnet-20240620-v1:0") rt := NewRouter(stubCredentialResolver{ok: false}, nil, nil, grantAll{}, nil, nil, nil) _, err := rt.ConverseStream(context.Background(), "000000000001", "anthropic.claude-3-5-sonnet-20240620-v1:0", converseStreamInput()) require.Error(t, err) @@ -237,6 +239,7 @@ func TestRouter_ConverseStream_SelfHostGatedAtCapacity(t *testing.T) { // never ThrottlingException — the managed-provider branch must never reach // admitSelfHost. func TestRouter_Converse_AnthropicPathIsNeverGated(t *testing.T) { + withProviderCatalogEntry(t, "anthropic.claude-3-5-sonnet-20240620-v1:0") rt := NewRouter(stubCredentialResolver{ok: false}, nil, nil, grantAll{}, nil, nil, nil) const n = 20 diff --git a/spinifex/gateway/spinifex.go b/spinifex/gateway/spinifex.go index fbba9907c..61fc5b1e2 100644 --- a/spinifex/gateway/spinifex.go +++ b/spinifex/gateway/spinifex.go @@ -22,6 +22,7 @@ var spinifexAdminActions = map[string]bool{ "GrantModelAccess": true, "RevokeModelAccess": true, "ListModelAccess": true, + "ListOchreCatalog": true, } func (gw *GatewayConfig) Spinifex_Request(w http.ResponseWriter, r *http.Request) error { @@ -111,6 +112,11 @@ func (gw *GatewayConfig) Spinifex_Request(w http.ResponseWriter, r *http.Request if models, err = gw.BedrockAccessAdmin.List(ctx, targetAccount); err == nil { output = gateway_bedrock.ModelAccessList{AccountID: targetAccount, ModelIDs: models} } + case "ListOchreCatalog": + var entries []gateway_bedrock.AdminCatalogEntry + if entries, err = gateway_bedrock.AdminCatalog(ctx, accountID, gw.bedrockResolver(), gw.bedrockAccessResolver()); err == nil { + output = gateway_bedrock.AdminCatalogList{Entries: entries} + } default: return errors.New(awserrors.ErrorInvalidAction) } diff --git a/tests/e2e/bedrock/bedrock_test.go b/tests/e2e/bedrock/bedrock_test.go index 41076f305..a8f3ee157 100644 --- a/tests/e2e/bedrock/bedrock_test.go +++ b/tests/e2e/bedrock/bedrock_test.go @@ -22,22 +22,15 @@ const ( modelBogus = "does.not-exist-v1:0" ) -// TestGetFoundationModel round-trips a known model and asserts -// ResourceNotFoundException for an unknown one. Uses the Anthropic entry: it -// resolves unconditionally, unlike the self-host entry, which is gated on a -// weights snapshot being staged — see TestGetFoundationModelSelfHost. +// TestGetFoundationModel asserts ResourceNotFoundException for an unknown +// model ID. v1 ships no provider-tier entry that resolves unconditionally +// (modelAnthropic is not in the catalog at all — it returns the same +// ResourceNotFoundException as a genuinely unknown ID), so the positive, +// known-model path is covered separately by TestGetFoundationModelSelfHost, +// which is gated on a real weights snapshot being staged. func TestGetFoundationModel(t *testing.T) { f := requireBedrockFixture(t) - t.Run("known model", func(t *testing.T) { - out, err := f.AWS.Bedrock.GetFoundationModel(&bedrock.GetFoundationModelInput{ - ModelIdentifier: aws.String(modelAnthropic), - }) - require.NoError(t, err, "get-foundation-model %s", modelAnthropic) - require.NotNil(t, out.ModelDetails, "empty ModelDetails") - assert.Equal(t, modelAnthropic, aws.StringValue(out.ModelDetails.ModelId)) - }) - t.Run("unknown model", func(t *testing.T) { harness.ExpectError(t, "ResourceNotFoundException", func() error { _, e := f.AWS.Bedrock.GetFoundationModel(&bedrock.GetFoundationModelInput{ @@ -46,6 +39,15 @@ func TestGetFoundationModel(t *testing.T) { return e }) }) + + t.Run("provider-tier model ID is absent from v1", func(t *testing.T) { + harness.ExpectError(t, "ResourceNotFoundException", func() error { + _, e := f.AWS.Bedrock.GetFoundationModel(&bedrock.GetFoundationModelInput{ + ModelIdentifier: aws.String(modelAnthropic), + }) + return e + }) + }) } // TestGetFoundationModelSelfHost describes the self-host model directly. @@ -91,9 +93,11 @@ func TestConverseSelfHost(t *testing.T) { } // TestConverseAnthropic exercises the real Anthropic-direct Converse path. -// Gated behind OCHRE_E2E_ANTHROPIC=1 — it needs a live Anthropic API key -// configured on the cluster (OCHRE_ANTHROPIC_API_KEY or a per-account KV entry). +// v1 ships no provider-tier catalog entry, so modelAnthropic cannot resolve +// on any cluster; this always skips regardless of OCHRE_E2E_ANTHROPIC until a +// later phase re-enables the provider-direct tier. func TestConverseAnthropic(t *testing.T) { + t.Skip("provider-direct tier not shipped in v1; anthropic entry is absent from the catalog") if os.Getenv("OCHRE_E2E_ANTHROPIC") != "1" { t.Skip("OCHRE_E2E_ANTHROPIC!=1; skipping real Anthropic Converse") } @@ -150,10 +154,12 @@ func TestInvokeModelSelfHost(t *testing.T) { } // TestInvokeModelAnthropic exercises the real Anthropic-direct InvokeModel -// path with the Bedrock-native Claude Messages request/response shape. Gated -// behind OCHRE_E2E_ANTHROPIC=1 — it needs a live Anthropic API key configured -// on the cluster (OCHRE_ANTHROPIC_API_KEY or a per-account KV entry). +// path with the Bedrock-native Claude Messages request/response shape. v1 +// ships no provider-tier catalog entry, so modelAnthropic cannot resolve on +// any cluster; this always skips regardless of OCHRE_E2E_ANTHROPIC until a +// later phase re-enables the provider-direct tier. func TestInvokeModelAnthropic(t *testing.T) { + t.Skip("provider-direct tier not shipped in v1; anthropic entry is absent from the catalog") if os.Getenv("OCHRE_E2E_ANTHROPIC") != "1" { t.Skip("OCHRE_E2E_ANTHROPIC!=1; skipping real Anthropic InvokeModel") }