mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-20 19:25:52 +00:00
Josh Siviour
a74052fd6f
The agent resolved the container runtime once, in New. A containerd socket that was not accepting during the first second of boot left the agent runtime-less for the life of the process, and nothing retried. It registered and heartbeated normally, so the scheduler saw a healthy instance and kept placing tasks that died immediately at the nil runtime check. The runtime now lives behind a gate that a watch loop keeps current: it dials while the gate is empty, and probes with a container list while it is full, clearing the gate when the probe fails so the next tick re-dials. That covers a socket that is not ready yet at boot and a containerd restarted underneath a running agent alike, and everything that used the puller or the runner reads through the gate, so a runtime arriving late is picked up without a restart. An agent with no runtime advertises no capacity. Registration and the heartbeat report zero CPU and zero memory while the gate is empty, so placement skips the host, and the control plane reads that pair of zeros as an involuntary drain: the instance becomes DRAINING with agentConnected false and zero registered resources, which is visible through DescribeContainerInstances rather than only in the node's log. Capacity returning clears it through the branch a reaped instance already used. A host with no CPU cannot exist, so the zero needs no new wire field. An operator drain is no longer cleared by a bus registration either, which it previously was. A deployment whose tasks keep failing to start now backs off. Deployment gains NextLaunchAt, and past the failure count the circuit breaker already tracks, the reconciler widens the gap between launch passes up to a cap. It is keyed on that counter but applies whether or not the breaker is enabled, because AWS leaves it off by default and an unbraked service relaunches a task that cannot start on every pass, forever.
123 lines
3.3 KiB
Go
123 lines
3.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
|
|
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
|
)
|
|
|
|
// TestRuntimeGate_DialsUntilTheSocketAnswers is the boot case the agent used to
|
|
// lose: the node starts containerd and the agent together, so the first dials
|
|
// fail. Resolving once left the agent runtime-less for the life of the process.
|
|
func TestRuntimeGate_DialsUntilTheSocketAnswers(t *testing.T) {
|
|
g := newRuntimeGate(nil)
|
|
if g.available() {
|
|
t.Fatal("empty gate reported a runtime")
|
|
}
|
|
|
|
rt := &ctrruntime.FakePuller{}
|
|
dials := 0
|
|
dial := func() (ctrruntime.Runtime, error) {
|
|
dials++
|
|
if dials < 3 {
|
|
return nil, errors.New("dial containerd: no such file or directory")
|
|
}
|
|
return rt, nil
|
|
}
|
|
|
|
for range 2 {
|
|
g.step(t.Context(), dial)
|
|
if g.available() {
|
|
t.Fatalf("gate resolved a runtime after %d failed dials", dials)
|
|
}
|
|
}
|
|
|
|
g.step(t.Context(), dial)
|
|
if g.get() != rt {
|
|
t.Fatalf("gate did not pick up the runtime once the socket answered (dials=%d)", dials)
|
|
}
|
|
}
|
|
|
|
// TestRuntimeGate_DropsARuntimeThatStopsAnswering covers a containerd restarted
|
|
// underneath a live agent: the held runtime stops answering, so the gate clears
|
|
// it and the next pass re-dials rather than handing out a dead client forever.
|
|
func TestRuntimeGate_DropsARuntimeThatStopsAnswering(t *testing.T) {
|
|
dead := &ctrruntime.FakePuller{ListErr: errors.New("connection refused")}
|
|
fresh := &ctrruntime.FakePuller{}
|
|
g := newRuntimeGate(dead)
|
|
|
|
dial := func() (ctrruntime.Runtime, error) { return fresh, nil }
|
|
|
|
g.step(t.Context(), dial)
|
|
if g.available() {
|
|
t.Fatal("gate kept a runtime whose probe failed")
|
|
}
|
|
if !dead.Closed {
|
|
t.Error("displaced runtime was not closed")
|
|
}
|
|
|
|
g.step(t.Context(), dial)
|
|
if g.get() != fresh {
|
|
t.Fatal("gate did not re-dial after clearing")
|
|
}
|
|
}
|
|
|
|
// TestRuntimeGate_KeepsAHealthyRuntime guards the other direction: a probe that
|
|
// succeeds must not churn the runtime, or every pass would drop a working client.
|
|
func TestRuntimeGate_KeepsAHealthyRuntime(t *testing.T) {
|
|
rt := &ctrruntime.FakePuller{}
|
|
g := newRuntimeGate(rt)
|
|
|
|
dial := func() (ctrruntime.Runtime, error) {
|
|
t.Error("dialled while holding a healthy runtime")
|
|
return nil, errors.New("unexpected dial")
|
|
}
|
|
g.step(t.Context(), dial)
|
|
|
|
if g.get() != rt {
|
|
t.Fatal("gate dropped a runtime whose probe succeeded")
|
|
}
|
|
if rt.Closed {
|
|
t.Error("healthy runtime was closed")
|
|
}
|
|
}
|
|
|
|
// TestRuntimeGate_ClearIsIdempotent keeps Stop safe to call on an agent that
|
|
// never resolved a runtime.
|
|
func TestRuntimeGate_ClearIsIdempotent(t *testing.T) {
|
|
g := newRuntimeGate(nil)
|
|
if g.clear() {
|
|
t.Error("clear reported dropping a runtime the gate never held")
|
|
}
|
|
|
|
rt := &ctrruntime.FakePuller{}
|
|
g.set(rt)
|
|
if !g.clear() {
|
|
t.Error("clear did not report dropping the held runtime")
|
|
}
|
|
if !rt.Closed {
|
|
t.Error("cleared runtime was not closed")
|
|
}
|
|
if g.clear() {
|
|
t.Error("second clear reported dropping a runtime")
|
|
}
|
|
}
|
|
|
|
// TestRuntimeGate_WatchStopsWithContext keeps the watch loop from outliving the
|
|
// agent it belongs to.
|
|
func TestRuntimeGate_WatchStopsWithContext(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
g := newRuntimeGate(nil)
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
g.watch(ctx, func() (ctrruntime.Runtime, error) { return nil, errors.New("no socket") }, 0)
|
|
close(done)
|
|
}()
|
|
|
|
cancel()
|
|
<-done
|
|
}
|