mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-22 04:07:04 +00:00
feat(bedrock): Ochre runtime MVP — Converse over self-host + Anthropic-direct (#520)
* feat(bedrock): Ochre runtime MVP — Converse over self-host + Anthropic-direct Add the AWS-Bedrock-compatible gateway services (bedrock, bedrock-runtime) following the EKS REST-JSON pattern: ListFoundationModels / GetFoundationModel over a static connectivity-tiered catalog, and non-streaming Converse routed per modelId to one of two provider backends — self-hosted vLLM (OpenAI wire) and Anthropic-direct (Anthropic Messages wire). Per-account provider API keys are a master-key-encrypted JetStream KV secret with an optional platform default; the advertised catalog gates external models on a resolvable credential. Self-host endpoints are pinned via config (OCHRE_VLLM_ENDPOINTS) behind an EndpointResolver seam. Adds gateway/bedrock/ (catalog, provider router, Anthropic + vLLM adapters, credential store), awserrors REST-JSON codes, and the bedrock E2E suite (list/get light; Converse gated on live backends). Daemon/NATS control plane and dynamic endpoint lifecycle are deferred to Phase 3. * test(bedrock): cover dispatch, router, credentials KV + endpoint config Add unit coverage for the Ochre Phase 1 diff: Bedrock_Request / BedrockRuntime_Request end-to-end dispatch, provider router, JSON response/error writers, upstream-status mapping, the credential-store KV round-trip, the Anthropic bound-provider path, and parseBedrockEndpoints. Lifts PR diff coverage above the 60% gate.
This commit is contained in:
Josh Siviour
+4
-3
@@ -1,4 +1,4 @@
|
||||
PKG ?= ./cert/... ./lb/... ./eks/... ./ecs/... ./iam/... ./ecr/... ./multinode/... ./baremetal/... ./reboot/... ./tofu/...
|
||||
PKG ?= ./cert/... ./lb/... ./eks/... ./ecs/... ./iam/... ./ecr/... ./bedrock/... ./multinode/... ./baremetal/... ./reboot/... ./tofu/...
|
||||
ARTIFACT_DIR ?= /tmp/spinifex-e2e-artifacts
|
||||
TIMEOUT ?= 30m
|
||||
MODE ?= single
|
||||
@@ -11,9 +11,9 @@ SUITE_TAG ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG))))
|
||||
# named suite into BIN_DIR/<suite>.test so CI can ship binaries (not source +
|
||||
# go toolchain) to the remote VM.
|
||||
BIN_DIR ?= $(CURDIR)/_bin
|
||||
SUITES_BUILD ?= cert lb eks ecs single iam ecr gpu natuplink
|
||||
SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink
|
||||
|
||||
.PHONY: e2e e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-all clean e2e-build
|
||||
.PHONY: e2e e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-all clean e2e-build
|
||||
|
||||
ifeq ($(GOTESTSUM),1)
|
||||
E2E_CMD = gotestsum \
|
||||
@@ -39,6 +39,7 @@ e2e-ecs: ; $(MAKE) e2e PKG=./ecs/...
|
||||
e2e-single: ; $(MAKE) e2e PKG=./single/...
|
||||
e2e-iam: ; $(MAKE) e2e PKG=./iam/...
|
||||
e2e-ecr: ; $(MAKE) e2e PKG=./ecr/...
|
||||
e2e-bedrock: ; $(MAKE) e2e PKG=./bedrock/...
|
||||
e2e-multinode: ; $(MAKE) e2e PKG=./multinode/... MODE=multinode
|
||||
e2e-bm: ; $(MAKE) e2e PKG=./baremetal/... MODE=baremetal
|
||||
e2e-reboot: ; $(MAKE) e2e PKG=./reboot/...
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
//go:build e2e
|
||||
|
||||
package bedrock
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/bedrock"
|
||||
"github.com/aws/aws-sdk-go/service/bedrockruntime"
|
||||
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Catalog model IDs mirror the static Phase-1 catalog in gateway_bedrock.
|
||||
const (
|
||||
modelSelfHostLlama = "meta.llama3-70b-instruct-v1:0"
|
||||
modelAnthropic = "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
||||
modelBogus = "does.not-exist-v1:0"
|
||||
)
|
||||
|
||||
// TestListFoundationModels confirms the self-host model is always advertised;
|
||||
// the Anthropic model is only asserted on when present, since its advertisement
|
||||
// depends on cluster credential configuration.
|
||||
func TestListFoundationModels(t *testing.T) {
|
||||
f := requireBedrockFixture(t)
|
||||
|
||||
out, err := f.AWS.Bedrock.ListFoundationModels(&bedrock.ListFoundationModelsInput{})
|
||||
require.NoError(t, err, "list-foundation-models")
|
||||
|
||||
ids := make(map[string]*bedrock.FoundationModelSummary, len(out.ModelSummaries))
|
||||
for _, m := range out.ModelSummaries {
|
||||
ids[aws.StringValue(m.ModelId)] = m
|
||||
}
|
||||
|
||||
assert.Contains(t, ids, modelSelfHostLlama, "self-host model must always be advertised")
|
||||
|
||||
if anthropic, ok := ids[modelAnthropic]; ok {
|
||||
assert.Equal(t, "Anthropic", aws.StringValue(anthropic.ProviderName),
|
||||
"anthropic model advertised with unexpected provider name")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetFoundationModel round-trips a known model and asserts
|
||||
// ResourceNotFoundException for an unknown one.
|
||||
func TestGetFoundationModel(t *testing.T) {
|
||||
f := requireBedrockFixture(t)
|
||||
|
||||
t.Run("known model", func(t *testing.T) {
|
||||
out, err := f.AWS.Bedrock.GetFoundationModel(&bedrock.GetFoundationModelInput{
|
||||
ModelIdentifier: aws.String(modelSelfHostLlama),
|
||||
})
|
||||
require.NoError(t, err, "get-foundation-model %s", modelSelfHostLlama)
|
||||
require.NotNil(t, out.ModelDetails, "empty ModelDetails")
|
||||
assert.Equal(t, modelSelfHostLlama, aws.StringValue(out.ModelDetails.ModelId))
|
||||
})
|
||||
|
||||
t.Run("unknown model", func(t *testing.T) {
|
||||
harness.ExpectError(t, "ResourceNotFoundException", func() error {
|
||||
_, e := f.AWS.Bedrock.GetFoundationModel(&bedrock.GetFoundationModelInput{
|
||||
ModelIdentifier: aws.String(modelBogus),
|
||||
})
|
||||
return e
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// TestConverseSelfHost exercises the real vLLM Converse path. Gated behind
|
||||
// OCHRE_E2E_SELFHOST=1 — it needs a GPU-backed vLLM endpoint configured on the
|
||||
// gateway via OCHRE_VLLM_ENDPOINTS at boot.
|
||||
func TestConverseSelfHost(t *testing.T) {
|
||||
if os.Getenv("OCHRE_E2E_SELFHOST") != "1" {
|
||||
t.Skip("OCHRE_E2E_SELFHOST!=1; skipping real self-host Converse")
|
||||
}
|
||||
f := requireBedrockFixture(t)
|
||||
|
||||
out, err := f.AWS.BedrockRuntime.Converse(&bedrockruntime.ConverseInput{
|
||||
ModelId: aws.String(modelSelfHostLlama),
|
||||
Messages: []*bedrockruntime.Message{{
|
||||
Role: aws.String(bedrockruntime.ConversationRoleUser),
|
||||
Content: []*bedrockruntime.ContentBlock{{Text: aws.String("Say hello in one word.")}},
|
||||
}},
|
||||
InferenceConfig: &bedrockruntime.InferenceConfiguration{MaxTokens: aws.Int64(64)},
|
||||
})
|
||||
require.NoError(t, err, "converse self-host")
|
||||
require.NotNil(t, out.Output, "nil output")
|
||||
require.NotNil(t, out.Output.Message, "nil output message")
|
||||
assert.NotEmpty(t, converseText(out.Output.Message), "empty assistant text")
|
||||
require.NotNil(t, out.Usage, "nil usage")
|
||||
assert.Greater(t, aws.Int64Value(out.Usage.OutputTokens), int64(0), "OutputTokens must be > 0")
|
||||
}
|
||||
|
||||
// TestConverseAnthropic exercises the real Anthropic-direct Converse path.
|
||||
// Gated behind OCHRE_E2E_ANTHROPIC=1 — it needs a live Anthropic API key
|
||||
// configured on the cluster (OCHRE_ANTHROPIC_API_KEY or a per-account KV entry).
|
||||
func TestConverseAnthropic(t *testing.T) {
|
||||
if os.Getenv("OCHRE_E2E_ANTHROPIC") != "1" {
|
||||
t.Skip("OCHRE_E2E_ANTHROPIC!=1; skipping real Anthropic Converse")
|
||||
}
|
||||
f := requireBedrockFixture(t)
|
||||
|
||||
out, err := f.AWS.BedrockRuntime.Converse(&bedrockruntime.ConverseInput{
|
||||
ModelId: aws.String(modelAnthropic),
|
||||
Messages: []*bedrockruntime.Message{{
|
||||
Role: aws.String(bedrockruntime.ConversationRoleUser),
|
||||
Content: []*bedrockruntime.ContentBlock{{Text: aws.String("Say hello in one word.")}},
|
||||
}},
|
||||
InferenceConfig: &bedrockruntime.InferenceConfiguration{MaxTokens: aws.Int64(64)},
|
||||
})
|
||||
require.NoError(t, err, "converse anthropic")
|
||||
require.NotNil(t, out.Output, "nil output")
|
||||
require.NotNil(t, out.Output.Message, "nil output message")
|
||||
assert.NotEmpty(t, converseText(out.Output.Message), "empty assistant text")
|
||||
require.NotNil(t, out.Usage, "nil usage")
|
||||
assert.Greater(t, aws.Int64Value(out.Usage.InputTokens), int64(0), "InputTokens must be > 0")
|
||||
assert.Greater(t, aws.Int64Value(out.Usage.OutputTokens), int64(0), "OutputTokens must be > 0")
|
||||
}
|
||||
|
||||
// converseText concatenates the text content blocks of a Converse output message.
|
||||
func converseText(msg *bedrockruntime.Message) string {
|
||||
var text string
|
||||
for _, c := range msg.Content {
|
||||
if c != nil && c.Text != nil {
|
||||
text += *c.Text
|
||||
}
|
||||
}
|
||||
return text
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
//go:build e2e
|
||||
|
||||
// Package bedrock is the Ochre (AWS-Bedrock) E2E suite. Hermetic API-shape and
|
||||
// provider-translation parity (Anthropic Messages / vLLM OpenAI wire formats)
|
||||
// is covered by the gateway_bedrock unit tests; this suite exercises
|
||||
// ListFoundationModels/GetFoundationModel against any running cluster and
|
||||
// gates real Converse calls on backend availability (self-host GPU or a live
|
||||
// Anthropic key), so CI parity holds without requiring either to be present.
|
||||
package bedrock
|
||||
|
||||
import (
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
||||
)
|
||||
|
||||
// Package-scoped singleton fixture, built lazily by requireBedrockFixture and
|
||||
// torn down by TestMain after the run.
|
||||
var (
|
||||
pkgFixOnce sync.Once
|
||||
pkgFix *Fixture
|
||||
pkgFixErr error
|
||||
)
|
||||
|
||||
// Fixture carries per-process state shared across every Test* in this package.
|
||||
type Fixture struct {
|
||||
Env *harness.Env
|
||||
AWS *harness.AWSClient
|
||||
Account string
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
// requireBedrockFixture returns the package-scoped Fixture, building it on
|
||||
// first call. Skips the calling test if SPINIFEX_E2E is unset.
|
||||
func requireBedrockFixture(t *testing.T) *Fixture {
|
||||
t.Helper()
|
||||
pkgFixOnce.Do(func() {
|
||||
if os.Getenv("SPINIFEX_E2E") == "" {
|
||||
return
|
||||
}
|
||||
env := harness.LoadEnv(t)
|
||||
awsCli := harness.NewAWSClient(t, env)
|
||||
pkgFix = &Fixture{
|
||||
Env: env,
|
||||
AWS: awsCli,
|
||||
Account: harness.IAMAccountID(t, awsCli),
|
||||
}
|
||||
})
|
||||
if pkgFixErr != nil {
|
||||
t.Fatalf("bedrock fixture init failed: %v", pkgFixErr)
|
||||
}
|
||||
if pkgFix == nil {
|
||||
t.Skip("bedrock fixture unavailable (SPINIFEX_E2E unset)")
|
||||
}
|
||||
return pkgFix
|
||||
}
|
||||
+24
-18
@@ -14,6 +14,8 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/acm"
|
||||
"github.com/aws/aws-sdk-go/service/bedrock"
|
||||
"github.com/aws/aws-sdk-go/service/bedrockruntime"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
|
||||
"github.com/aws/aws-sdk-go/service/ecr"
|
||||
@@ -30,15 +32,17 @@ type AWSClient struct {
|
||||
// EC2 is wrapped so RunInstances auto-serialises against cluster capacity
|
||||
// (see capacityRetryEC2). EC2Conf is the underlying SDK client shared by
|
||||
// both; kept for call sites that read or mutate its Config.
|
||||
EC2 ec2iface.EC2API
|
||||
EC2Conf *ec2.EC2
|
||||
ELBv2 *elbv2.ELBV2
|
||||
IAM *iam.IAM
|
||||
STS *sts.STS
|
||||
EKS *eks.EKS
|
||||
ACM *acm.ACM
|
||||
ECR *ecr.ECR
|
||||
ECS *ecs.ECS
|
||||
EC2 ec2iface.EC2API
|
||||
EC2Conf *ec2.EC2
|
||||
ELBv2 *elbv2.ELBV2
|
||||
IAM *iam.IAM
|
||||
STS *sts.STS
|
||||
EKS *eks.EKS
|
||||
ACM *acm.ACM
|
||||
ECR *ecr.ECR
|
||||
ECS *ecs.ECS
|
||||
Bedrock *bedrock.Bedrock
|
||||
BedrockRuntime *bedrockruntime.BedrockRuntime
|
||||
}
|
||||
|
||||
// NewAWSClient builds clients pointed at the spinifex gateway using the
|
||||
@@ -128,15 +132,17 @@ func newAWSClient(t *testing.T, env *Env, accessKey, secretKey, sessionToken str
|
||||
|
||||
rawEC2 := ec2.New(sess)
|
||||
return &AWSClient{
|
||||
EC2: &capacityRetryEC2{EC2API: rawEC2},
|
||||
EC2Conf: rawEC2,
|
||||
ELBv2: elbv2.New(sess),
|
||||
IAM: iam.New(sess),
|
||||
STS: sts.New(sess),
|
||||
EKS: eks.New(sess),
|
||||
ACM: acm.New(sess),
|
||||
ECR: ecr.New(sess),
|
||||
ECS: ecs.New(sess),
|
||||
EC2: &capacityRetryEC2{EC2API: rawEC2},
|
||||
EC2Conf: rawEC2,
|
||||
ELBv2: elbv2.New(sess),
|
||||
IAM: iam.New(sess),
|
||||
STS: sts.New(sess),
|
||||
EKS: eks.New(sess),
|
||||
ACM: acm.New(sess),
|
||||
ECR: ecr.New(sess),
|
||||
ECS: ecs.New(sess),
|
||||
Bedrock: bedrock.New(sess),
|
||||
BedrockRuntime: bedrockruntime.New(sess),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user