feat(daemon): make hostReserve env-configurable for dev capacity squeeze

Adds SPINIFEX_RESERVED_VCPU and SPINIFEX_RESERVED_MEM_GB env-var overrides
read at daemon startup. Invalid values are logged and ignored so a typo
doesn't silently widen the reserve. Unblocks the LB E2E on capacity-tight
dev hosts (8 vCPU box: default 2-vCPU reserve + 3x t3.nano test load
leaves 0 free for the sys.micro LB VM). Lays groundwork for the planned
operator-tunable [capacity] config.

bead: mulga-siv-78
This commit is contained in:
joshsiv-mulga
2026-05-18 11:32:07 +10:00
parent f902131841
commit e28183f309
3 changed files with 97 additions and 2 deletions
+3 -2
View File
@@ -408,11 +408,12 @@ func NewResourceManager(gpuModels []instancetypes.GPUModel, gpuMgr *gpu.Manager)
return nil, fmt.Errorf("detect system memory: %w", err)
}
reservedVCPU, reservedMem, err := applyHostReserve(defaultHostReserve, numCPU, totalMemGB)
reserve := resolveHostReserve(os.Getenv)
reservedVCPU, reservedMem, err := applyHostReserve(reserve, numCPU, totalMemGB)
if err != nil {
slog.Error("host below minimum reserve — daemon refuses to start",
"err", err, "hostVCPU", numCPU, "hostMemGB", totalMemGB,
"reserveVCPU", defaultHostReserve.vCPU, "reserveMemGB", defaultHostReserve.memGB)
"reserveVCPU", reserve.vCPU, "reserveMemGB", reserve.memGB)
return nil, fmt.Errorf("validate host reserve: %w", err)
}
+25
View File
@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"log/slog"
"strconv"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/spinifex/types"
@@ -27,6 +28,30 @@ type hostReserve struct {
// the daemon itself needs little.
var defaultHostReserve = hostReserve{vCPU: 2, memGB: 2.0}
// resolveHostReserve returns defaultHostReserve, with vCPU/memGB overridden
// when SPINIFEX_RESERVED_VCPU / SPINIFEX_RESERVED_MEM_GB are set to valid
// non-negative values. Invalid values are logged and ignored — keeps a
// typo'd env from silently widening the reserve. Intended as a stop-gap
// until the planned operator-tunable [capacity] config lands.
func resolveHostReserve(getenv func(string) string) hostReserve {
r := defaultHostReserve
if v := getenv("SPINIFEX_RESERVED_VCPU"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n >= 0 {
r.vCPU = n
} else {
slog.Warn("ignoring SPINIFEX_RESERVED_VCPU", "value", v, "err", err)
}
}
if v := getenv("SPINIFEX_RESERVED_MEM_GB"); v != "" {
if f, err := strconv.ParseFloat(v, 64); err == nil && f >= 0 {
r.memGB = f
} else {
slog.Warn("ignoring SPINIFEX_RESERVED_MEM_GB", "value", v, "err", err)
}
}
return r
}
// minHostMemHeadroomGB is the minimum schedulable memory we require above
// the reserve, so a host that just meets the reserve still has a small
// amount left to launch the smallest guest type.
+69
View File
@@ -373,3 +373,72 @@ func TestAllocateForLaunch(t *testing.T) {
})
}
}
func TestResolveHostReserve(t *testing.T) {
tests := []struct {
name string
env map[string]string
wantVCPU int
wantMem float64
}{
{
name: "no env returns default",
env: map[string]string{},
wantVCPU: defaultHostReserve.vCPU,
wantMem: defaultHostReserve.memGB,
},
{
name: "vCPU override",
env: map[string]string{"SPINIFEX_RESERVED_VCPU": "1"},
wantVCPU: 1,
wantMem: defaultHostReserve.memGB,
},
{
name: "memory override",
env: map[string]string{"SPINIFEX_RESERVED_MEM_GB": "0.5"},
wantVCPU: defaultHostReserve.vCPU,
wantMem: 0.5,
},
{
name: "both overrides",
env: map[string]string{
"SPINIFEX_RESERVED_VCPU": "0",
"SPINIFEX_RESERVED_MEM_GB": "1.5",
},
wantVCPU: 0,
wantMem: 1.5,
},
{
name: "invalid vCPU falls back to default",
env: map[string]string{"SPINIFEX_RESERVED_VCPU": "garbage"},
wantVCPU: defaultHostReserve.vCPU,
wantMem: defaultHostReserve.memGB,
},
{
name: "negative vCPU falls back to default",
env: map[string]string{"SPINIFEX_RESERVED_VCPU": "-1"},
wantVCPU: defaultHostReserve.vCPU,
wantMem: defaultHostReserve.memGB,
},
{
name: "invalid memory falls back to default",
env: map[string]string{"SPINIFEX_RESERVED_MEM_GB": "not-a-number"},
wantVCPU: defaultHostReserve.vCPU,
wantMem: defaultHostReserve.memGB,
},
{
name: "negative memory falls back to default",
env: map[string]string{"SPINIFEX_RESERVED_MEM_GB": "-0.5"},
wantVCPU: defaultHostReserve.vCPU,
wantMem: defaultHostReserve.memGB,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := resolveHostReserve(func(k string) string { return tc.env[k] })
assert.Equal(t, tc.wantVCPU, got.vCPU)
assert.InDelta(t, tc.wantMem, got.memGB, 0.001)
})
}
}