From 822db926feece23bf5d1017052dcbe208f67efe0 Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Sat, 11 Jul 2026 00:02:03 +1000 Subject: [PATCH] feat(observability): surface AWS error message + show schedulable node capacity (#505) - 4ncdg: NATSRequest now round-trips the AWS error Message, not just the sanitized Code. Server sends GenerateErrorPayloadWithMessage(code, err.Error()); the client returns the Message when present so the CLI prints the real reason (e.g. "snapshot not found; refusing to restore") instead of a bare "ServerInternal". One code path fixes every NATS-driven command. - fbmhl: the UI node view rendered raw host totals (vCPU:80, 0/125.5GB), ignoring the daemon reserve. It now shows schedulable = total - reserved, the capacity a tenant can actually consume. reserved_vcpu/reserved_mem_gb added to the NodeInfo type (already present in the API payload). --- .../spinifexui/frontend/src/queries/admin.ts | 2 + .../frontend/src/routes/_auth/nodes.tsx | 47 +++++++++++-------- spinifex/utils/nats.go | 5 +- spinifex/utils/nats_test.go | 21 +++++++++ spinifex/utils/utils.go | 11 +++++ 5 files changed, 65 insertions(+), 21 deletions(-) diff --git a/spinifex/services/spinifexui/frontend/src/queries/admin.ts b/spinifex/services/spinifexui/frontend/src/queries/admin.ts index 5d2875788..3bbad2993 100644 --- a/spinifex/services/spinifexui/frontend/src/queries/admin.ts +++ b/spinifex/services/spinifexui/frontend/src/queries/admin.ts @@ -47,6 +47,8 @@ interface NodeInfo { vm_count: number total_vcpu: number total_mem_gb: number + reserved_vcpu: number + reserved_mem_gb: number alloc_vcpu: number alloc_mem_gb: number total_gpus: number diff --git a/spinifex/services/spinifexui/frontend/src/routes/_auth/nodes.tsx b/spinifex/services/spinifexui/frontend/src/routes/_auth/nodes.tsx index 5f9fe7481..d802699fd 100644 --- a/spinifex/services/spinifexui/frontend/src/routes/_auth/nodes.tsx +++ b/spinifex/services/spinifexui/frontend/src/routes/_auth/nodes.tsx @@ -301,31 +301,38 @@ function ResourceTable({ nodes }: { nodes: NodeInfo[] }) { Name - CPU (used/total) - MEM (used/total) + CPU (used/sched) + MEM (used/sched) GPU (used/total) EC2 - {nodes.map((node) => ( - - {node.node} - - {node.alloc_vcpu}/{node.total_vcpu} - - - {formatMemory(node.alloc_mem_gb)}/ - {formatMemory(node.total_mem_gb)} - - - {node.total_gpus > 0 - ? `${node.alloc_gpus}/${node.total_gpus}` - : "-"} - - {node.vm_count} - - ))} + {nodes.map((node) => { + // Schedulable = host total minus the daemon reserve; this is the + // capacity a tenant deployment can actually consume. + const schedVcpu = node.total_vcpu - (node.reserved_vcpu ?? 0) + const schedMemGb = node.total_mem_gb - (node.reserved_mem_gb ?? 0) + return ( + + + {node.node} + + + {node.alloc_vcpu}/{schedVcpu} + + + {formatMemory(node.alloc_mem_gb)}/{formatMemory(schedMemGb)} + + + {node.total_gpus > 0 + ? `${node.alloc_gpus}/${node.total_gpus}` + : "-"} + + {node.vm_count} + + ) + })} diff --git a/spinifex/utils/nats.go b/spinifex/utils/nats.go index 7fbeb63fe..2ae123d91 100644 --- a/spinifex/utils/nats.go +++ b/spinifex/utils/nats.go @@ -257,6 +257,9 @@ func NATSRequest[Out any](ctx context.Context, conn *nats.Conn, subject string, responseError, err := ValidateErrorPayload(msg.Data) if err != nil { + if responseError.Message != nil && *responseError.Message != "" { + return nil, errors.New(*responseError.Message) + } return nil, errors.New(*responseError.Code) } @@ -289,7 +292,7 @@ func ServeNATSRequestCtx[I any, O any](msg *nats.Msg, fn func(context.Context, * out, err := fn(ctx, input) if err != nil { MarkSpanError(span, err) - respondNATS(msg, GenerateErrorPayload(awserrors.ValidErrorCode(err.Error()))) + respondNATS(msg, GenerateErrorPayloadWithMessage(awserrors.ValidErrorCode(err.Error()), err.Error())) return } data, err := json.Marshal(out) diff --git a/spinifex/utils/nats_test.go b/spinifex/utils/nats_test.go index b4fdd984f..eb10edcb2 100644 --- a/spinifex/utils/nats_test.go +++ b/spinifex/utils/nats_test.go @@ -281,6 +281,27 @@ func TestNATSRequest_ErrorResponse(t *testing.T) { assert.Contains(t, err.Error(), "InvalidParameterValue") } +func TestNATSRequest_ErrorResponseSurfacesMessage(t *testing.T) { + ns := startTestNATSServer(t) + + nc, err := nats.Connect(ns.ClientURL()) + require.NoError(t, err) + defer nc.Close() + + // Code is sanitized to ServerInternal, but the actionable message must + // still reach the caller instead of the bare code. + const reason = "eks: snapshot \"snap-bogus\" not found; refusing to restore" + _, err = nc.Subscribe("test.fail.msg", func(msg *nats.Msg) { + msg.Respond(GenerateErrorPayloadWithMessage("ServerInternal", reason)) + }) + require.NoError(t, err) + + type Resp struct{} + _, err = NATSRequest[Resp](context.Background(), nc, "test.fail.msg", struct{}{}, 2*time.Second, "") + assert.Error(t, err) + assert.Equal(t, reason, err.Error()) +} + func TestNATSRequest_NoResponders(t *testing.T) { ns := startTestNATSServer(t) diff --git a/spinifex/utils/utils.go b/spinifex/utils/utils.go index 4164c3be4..9f72f359c 100644 --- a/spinifex/utils/utils.go +++ b/spinifex/utils/utils.go @@ -97,8 +97,19 @@ func GenerateIAMXMLPayload(action string, payload any) any { // GenerateErrorPayload serializes an ec2.ResponseError with the given code as JSON. func GenerateErrorPayload(code string) (jsonResponse []byte) { + return GenerateErrorPayloadWithMessage(code, "") +} + +// GenerateErrorPayloadWithMessage serializes an ec2.ResponseError carrying both +// the sanitized code and the original error message, so the client can surface +// the actionable reason instead of a bare code. Message is omitted when empty +// or identical to the code. +func GenerateErrorPayloadWithMessage(code, message string) (jsonResponse []byte) { var responseError ec2.ResponseError responseError.Code = aws.String(code) + if message != "" && message != code { + responseError.Message = aws.String(message) + } jsonResponse, err := json.Marshal(responseError) if err != nil { slog.Error("GenerateErrorPayload could not marshal JSON payload", "err", err)