Files
spinifex/cmd/ecs-agent/taskrun_test.go
T
Josh Siviour de64d6c63d feat(ecs): agent control-plane over the AWS gateway (drop direct NATS) (#419)
* feat(ecsgw): on-VM SigV4 client for the ECS agent→gateway channel

Adds internal/ecsgw, the twin of internal/eksgw: a TLS + SigV4 (service "ecs")
client the ecs-agent will use to reach the AWS gateway over HTTPS instead of
connecting to host NATS directly. Calls are AWS JSON 1.1 with an X-Amz-Target
action; the gateway relays onto ecs.* host-side, keeping NATS cluster-internal.

* feat(ecs): SubmitTaskStateChange gateway action → recordTaskState

Wires the AWS-API SubmitTaskStateChange action end to end (gateway action →
ecs.SubmitTaskStateChange NATS → daemon → Service.SubmitTaskStateChange) so a
gateway-routed agent reports task state without touching the NATS bus. The
handler maps the SDK input onto the same bus.TaskState shape and converges on
recordTaskState; the account is authoritative from the SigV4 caller, so an
instance cannot report state for another account's task.

* feat(ecs): durable assign KV inbox + PollAssignments gateway action

Scheduler writes task assigns into a per-instance KV inbox
(clusters/{cluster}/assignments/{instance}/{task}) instead of a
fire-and-forget core-NATS publish, and a new PollAssignments gateway
action lets a gateway-routed agent drain its inbox over TLS+SigV4.

At-least-once: an unacked assign is re-delivered on the next poll;
the agent acks accepted assigns (AckTaskIDs) and STOPPED transitions
reclaim stale inbox entries. The Poll response carries bus.Assign with
an RFC3339 time, so it is encoded with encoding/json (RawJSONActions)
rather than the jsonutil epoch-second marshaler.

* feat(ecs-agent): rewire control-plane off NATS onto the AWS gateway

The agent no longer connects to NATS. It registers, heartbeats, reports
task state, and receives assignments entirely through the AWS gateway
over TLS+SigV4, using seeded IAM credentials — keeping the NATS bus
host-internal and closing the trust-boundary gap (a tenant container
instance must not hold a shared bus token).

- controlPlane seam over ecsgw.Client replaces the NATS publisher:
  Register (RegisterContainerInstance), SubmitTaskState
  (SubmitTaskStateChange), PollAssignments.
- Heartbeat folds into a periodic idempotent re-register (refreshes
  LastSeen); bus.Heartbeat is dropped from the agent.
- subscribeAssign (NATS subscribe) becomes a poll loop: drain the
  inbox, dispatch each assign once, ack accepted taskIDs on the next
  poll (at-least-once).
- config: ECS_ACCESS_KEY / ECS_SECRET_KEY / ECS_POLL_INTERVAL added,
  ECS_NATS_URL removed; cloud-init initd seeds the IAM creds.

Tests move from the NATS publisher fake to a controlPlane fake.

* feat(ecs): cloud-init cred-seed + ecs-node bring-up script

ECS container instances are plain EC2 booted from the spinifex-ecs-node
AMI (AWS-faithful: no ECS-specific launch API). This renders the
cloud-init user-data that seeds /etc/spinifex-ecs/agent.env (gateway
URL/CA/region/cluster + IAM creds) and the gateway CA, then runs the
instance. The agent registers, heartbeats, polls assignments, and
reports task state entirely through the AWS gateway over TLS+SigV4 —
never NATS.

The guest dials the customer-facing gateway address (network advertise
IP), not the mgmt bridge, mirroring the EKS GatewayURL/AddonGatewayURL
split. Seeds the system key for now; scoped per-instance creds are a
follow-up (4g task-role endpoint).

* fix(ecs): reconcile dev merge — restore #418 ENI release + netns wiring

The 'Merge branch dev' resolution dropped code where Sprint 4d (#418,
awsvpc ENI-per-task) and the gateway control-plane rewrite (#419) touch
the same paths:

- recordTaskState STOPPED branch: reclaimTaskENI was clobbered by
  reclaimAssignInbox; both are needed (release the task ENI AND drain the
  KV inbox). Restores TestRecordTaskState_Awsvpc_ReleasesENIOnStopOnce.
- ecs-agent Agent: the netns field, its init, and the setupTaskNetns/
  teardownTaskNetns wrappers were lost when the controlPlane Agent struct
  won the merge, leaving taskrun.go calling undefined methods.
- eni_test assign assertion read the assign over NATS; #419 delivers it via
  the durable KV inbox, so it now polls via PollAssignments.
- taskrun_test netns cases used the removed historyPub fake; switched to
  fakeCP to match the controlPlane seam.
2026-06-25 18:42:24 +10:00

230 lines
6.9 KiB
Go

package main
import (
"context"
"errors"
"testing"
"time"
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
)
func testAssign() *bus.Assign {
return &bus.Assign{
AccountID: "123456789012",
ClusterName: "default",
InstanceID: "i-abc123",
TaskID: "t-001",
Containers: []bus.AssignContainer{
{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"},
Environment: map[string]string{"FOO": "bar"}},
},
}
}
func newRunAgent(cp controlPlane, rt *ctrruntime.FakePuller) *Agent {
return newAgent(config{}, testIdentity(), cp, rt, rt, nil)
}
// runTask with no runtime reports the task STOPPED instead of crashing.
func TestRunTask_NoRuntimeReportsStopped(t *testing.T) {
cp := &fakeCP{}
a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
a.runTask(context.Background(), testAssign())
st := cp.taskStates()
if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
t.Fatalf("want one STOPPED state, got %+v", st)
}
if st[0].Reason == "" {
t.Error("STOPPED state missing reason")
}
}
// Happy path: pull + run reports RUNNING with the container ID; Wait blocking
// (forced error) means no STOPPED overwrite, so we can assert the RUNNING frame.
func TestRunTask_PullRunReportsRunning(t *testing.T) {
cp := &fakeCP{}
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
a := newRunAgent(cp, rt)
as := testAssign()
a.runTask(context.Background(), as)
if len(rt.Pulls) != 1 || rt.Pulls[0].Ref != "registry/web:1" {
t.Fatalf("expected one pull of registry/web:1, got %+v", rt.Pulls)
}
if len(rt.Runs) != 1 || rt.Runs[0].Image != "registry/web:1" {
t.Fatalf("expected one run, got %+v", rt.Runs)
}
if rt.Runs[0].Labels["mulga.ecs.taskID"] != "t-001" {
t.Errorf("missing task label: %+v", rt.Runs[0].Labels)
}
st := cp.taskStates()
if len(st) == 0 || st[len(st)-1].LastStatus != bus.TaskStatusRunning {
t.Fatalf("want RUNNING final state, got %+v", st)
}
if len(st[len(st)-1].Containers) != 1 || st[len(st)-1].Containers[0].ContainerID != "t-001-web" {
t.Errorf("RUNNING state container wrong: %+v", st[len(st)-1].Containers)
}
}
// A pull failure reports STOPPED and never starts the container.
func TestRunTask_PullFailureReportsStopped(t *testing.T) {
cp := &fakeCP{}
rt := &ctrruntime.FakePuller{Err: errors.New("pull denied")}
a := newRunAgent(cp, rt)
a.runTask(context.Background(), testAssign())
if len(rt.Runs) != 0 {
t.Errorf("container should not run after pull failure, got %+v", rt.Runs)
}
st := cp.taskStates()
if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
t.Fatalf("want STOPPED, got %+v", st)
}
}
// A run failure reports STOPPED.
func TestRunTask_RunFailureReportsStopped(t *testing.T) {
cp := &fakeCP{}
rt := &ctrruntime.FakePuller{RunErr: errors.New("start boom")}
a := newRunAgent(cp, rt)
a.runTask(context.Background(), testAssign())
st := cp.taskStates()
if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
t.Fatalf("want STOPPED, got %+v", st)
}
}
// On container exit, waitContainer reports STOPPED with the exit code.
func TestRunTask_ExitReportsStoppedWithCode(t *testing.T) {
cp := &fakeCP{}
rt := &ctrruntime.FakePuller{WaitCode: 7}
a := newRunAgent(cp, rt)
a.runTask(context.Background(), testAssign())
deadline := time.After(time.Second)
for {
st := cp.taskStates()
if len(st) > 0 && st[len(st)-1].LastStatus == bus.TaskStatusStopped {
last := st[len(st)-1]
if len(last.Containers) != 1 || last.Containers[0].ExitCode == nil || *last.Containers[0].ExitCode != 7 {
t.Fatalf("want exit code 7, got %+v", last.Containers)
}
return
}
select {
case <-deadline:
t.Fatalf("no STOPPED state after exit; got %+v", cp.taskStates())
case <-time.After(2 * time.Millisecond):
}
}
}
// pollAssignments dispatches each assign once and acks it on the next poll.
func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
as := testAssign()
cp := &fakeCP{pollReplies: [][]bus.Assign{
{*as}, // poll 1: one pending assign
{*as}, // poll 2: still present (not yet acked when poll 2 was issued)
nil, // poll 3: gateway dropped it after the ack
}}
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
ctx, cancel := context.WithCancel(context.Background())
go a.pollAssignments(ctx)
// runTask reports RUNNING through the (mutex-guarded) control plane. With the
// assign returned on two consecutive polls, dedup means exactly one dispatch →
// exactly one RUNNING report.
deadline := time.After(time.Second)
for {
if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running >= 1 {
break
}
select {
case <-deadline:
t.Fatal("assign was never dispatched (no RUNNING report)")
case <-time.After(2 * time.Millisecond):
}
}
time.Sleep(20 * time.Millisecond)
cancel()
if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 1 {
t.Errorf("task dispatched %d times, want exactly 1 (dedup on taskID)", running)
}
// The taskID must appear in a later poll's ack list.
acked := false
for _, ack := range cp.acks() {
for _, id := range ack {
if id == as.TaskID {
acked = true
}
}
}
if !acked {
t.Errorf("task %s was never acked; acks=%v", as.TaskID, cp.acks())
}
}
func countStatus(states []bus.TaskState, taskID, status string) int {
n := 0
for _, st := range states {
if st.TaskID == taskID && st.LastStatus == status {
n++
}
}
return n
}
// awsvpc: an assign carrying an ENI MAC builds the task netns and passes its
// path to the container RunSpec.
func TestRunTask_AwsvpcBuildsNetns(t *testing.T) {
cp := &fakeCP{}
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
a := newRunAgent(cp, rt)
f := &fakeNetRunner{linkOut: linkWithENI}
a.netns = newTestNetns(f)
as := testAssign()
as.ENIMacAddress = "52:54:00:de:ad:01"
a.runTask(context.Background(), as)
if !f.sawAny("ip netns add ecs-t-001") {
t.Fatalf("expected task netns build, got %v", f.joined())
}
if len(rt.Runs) != 1 || rt.Runs[0].NetnsPath != netnsPathFor("t-001") {
t.Fatalf("want NetnsPath %q on RunSpec, got %+v", netnsPathFor("t-001"), rt.Runs)
}
}
// bridge/host: no ENI MAC means no netns is built and the container runs in the
// host (VM) netns (empty NetnsPath).
func TestRunTask_NoMacSkipsNetns(t *testing.T) {
cp := &fakeCP{}
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
a := newRunAgent(cp, rt)
f := &fakeNetRunner{linkOut: linkWithENI}
a.netns = newTestNetns(f)
a.runTask(context.Background(), testAssign())
if len(f.joined()) != 0 {
t.Fatalf("expected no netns commands, got %v", f.joined())
}
if len(rt.Runs) != 1 || rt.Runs[0].NetnsPath != "" {
t.Fatalf("want empty NetnsPath, got %+v", rt.Runs)
}
}
func TestContainerID(t *testing.T) {
if got := containerID("t-1", "web"); got != "t-1-web" {
t.Errorf("containerID = %q, want t-1-web", got)
}
}