gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-16 01:22:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(observability): surface AWS error message + show schedulable node capacity (#505)

Browse Source 
- 4ncdg: NATSRequest now round-trips the AWS error Message, not just the
  sanitized Code. Server sends GenerateErrorPayloadWithMessage(code,
  err.Error()); the client returns the Message when present so the CLI prints
  the real reason (e.g. "snapshot not found; refusing to restore") instead of
  a bare "ServerInternal". One code path fixes every NATS-driven command.
- fbmhl: the UI node view rendered raw host totals (vCPU:80, 0/125.5GB),
  ignoring the daemon reserve. It now shows schedulable = total - reserved,
  the capacity a tenant can actually consume. reserved_vcpu/reserved_mem_gb
  added to the NodeInfo type (already present in the API payload).
This commit is contained in:
Josh Siviour
2026-07-11 00:02:03 +10:00
committed by GitHub
parent 183a08353e
commit 822db926fe
5 changed files with 65 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
spinifex/services/spinifexui/frontend/src/queries/admin.ts
+2
View File
@@ -47,6 +47,8 @@ interface NodeInfo {
vm_count: number
total_vcpu: number
total_mem_gb: number
reserved_vcpu: number
reserved_mem_gb: number
alloc_vcpu: number
alloc_mem_gb: number
total_gpus: number
spinifex/services/spinifexui/frontend/src/routes/_auth/nodes.tsx
+27 -20
View File
@@ -301,31 +301,38 @@ function ResourceTable({ nodes }: { nodes: NodeInfo[] }) {
<thead>
<tr className="border-b text-left text-muted-foreground">
<th className="pr-4 pb-1 font-medium">Name</th>
<th className="pr-4 pb-1 font-medium">CPU (used/total)</th>
<th className="pr-4 pb-1 font-medium">MEM (used/total)</th>
<th className="pr-4 pb-1 font-medium">CPU (used/sched)</th>
<th className="pr-4 pb-1 font-medium">MEM (used/sched)</th>
<th className="pr-4 pb-1 font-medium">GPU (used/total)</th>
<th className="pb-1 font-medium">EC2</th>
</tr>
</thead>
<tbody>
{nodes.map((node) => (
<tr key={node.node} className="border-b last:border-0">
<td className="py-1.5 pr-4 font-mono font-medium">{node.node}</td>
<td className="py-1.5 pr-4 font-mono">
{node.alloc_vcpu}/{node.total_vcpu}
</td>
<td className="py-1.5 pr-4 font-mono">
{formatMemory(node.alloc_mem_gb)}/
{formatMemory(node.total_mem_gb)}
</td>
<td className="py-1.5 pr-4 font-mono">
{node.total_gpus > 0
? `${node.alloc_gpus}/${node.total_gpus}`
: "-"}
</td>
<td className="py-1.5">{node.vm_count}</td>
</tr>
))}
{nodes.map((node) => {
// Schedulable = host total minus the daemon reserve; this is the
// capacity a tenant deployment can actually consume.
const schedVcpu = node.total_vcpu - (node.reserved_vcpu ?? 0)
const schedMemGb = node.total_mem_gb - (node.reserved_mem_gb ?? 0)
return (
<tr key={node.node} className="border-b last:border-0">
<td className="py-1.5 pr-4 font-mono font-medium">
{node.node}
</td>
<td className="py-1.5 pr-4 font-mono">
{node.alloc_vcpu}/{schedVcpu}
</td>
<td className="py-1.5 pr-4 font-mono">
{formatMemory(node.alloc_mem_gb)}/{formatMemory(schedMemGb)}
</td>
<td className="py-1.5 pr-4 font-mono">
{node.total_gpus > 0
? `${node.alloc_gpus}/${node.total_gpus}`
: "-"}
</td>
<td className="py-1.5">{node.vm_count}</td>
</tr>
)
})}
</tbody>
</table>
</div>
spinifex/utils/nats.go
+4 -1
View File
@@ -257,6 +257,9 @@ func NATSRequest[Out any](ctx context.Context, conn *nats.Conn, subject string,
responseError, err := ValidateErrorPayload(msg.Data)
if err != nil {
if responseError.Message != nil && *responseError.Message != "" {
return nil, errors.New(*responseError.Message)
}
return nil, errors.New(*responseError.Code)
}
@@ -289,7 +292,7 @@ func ServeNATSRequestCtx[I any, O any](msg *nats.Msg, fn func(context.Context, *
out, err := fn(ctx, input)
if err != nil {
MarkSpanError(span, err)
respondNATS(msg, GenerateErrorPayload(awserrors.ValidErrorCode(err.Error())))
respondNATS(msg, GenerateErrorPayloadWithMessage(awserrors.ValidErrorCode(err.Error()), err.Error()))
return
}
data, err := json.Marshal(out)
spinifex/utils/nats_test.go
+21
View File
@@ -281,6 +281,27 @@ func TestNATSRequest_ErrorResponse(t *testing.T) {
assert.Contains(t, err.Error(), "InvalidParameterValue")
}
func TestNATSRequest_ErrorResponseSurfacesMessage(t *testing.T) {
ns := startTestNATSServer(t)
nc, err := nats.Connect(ns.ClientURL())
require.NoError(t, err)
defer nc.Close()
// Code is sanitized to ServerInternal, but the actionable message must
// still reach the caller instead of the bare code.
const reason = "eks: snapshot \"snap-bogus\" not found; refusing to restore"
_, err = nc.Subscribe("test.fail.msg", func(msg *nats.Msg) {
msg.Respond(GenerateErrorPayloadWithMessage("ServerInternal", reason))
})
require.NoError(t, err)
type Resp struct{}
_, err = NATSRequest[Resp](context.Background(), nc, "test.fail.msg", struct{}{}, 2*time.Second, "")
assert.Error(t, err)
assert.Equal(t, reason, err.Error())
}
func TestNATSRequest_NoResponders(t *testing.T) {
ns := startTestNATSServer(t)
spinifex/utils/utils.go
+11
View File
@@ -97,8 +97,19 @@ func GenerateIAMXMLPayload(action string, payload any) any {
// GenerateErrorPayload serializes an ec2.ResponseError with the given code as JSON.
func GenerateErrorPayload(code string) (jsonResponse []byte) {
return GenerateErrorPayloadWithMessage(code, "")
}
// GenerateErrorPayloadWithMessage serializes an ec2.ResponseError carrying both
// the sanitized code and the original error message, so the client can surface
// the actionable reason instead of a bare code. Message is omitted when empty
// or identical to the code.
func GenerateErrorPayloadWithMessage(code, message string) (jsonResponse []byte) {
var responseError ec2.ResponseError
responseError.Code = aws.String(code)
if message != "" && message != code {
responseError.Message = aws.String(message)
}
jsonResponse, err := json.Marshal(responseError)
if err != nil {
slog.Error("GenerateErrorPayload could not marshal JSON payload", "err", err)