mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-20 11:21:49 +00:00
Josh Siviour
a74052fd6f
The agent resolved the container runtime once, in New. A containerd socket that was not accepting during the first second of boot left the agent runtime-less for the life of the process, and nothing retried. It registered and heartbeated normally, so the scheduler saw a healthy instance and kept placing tasks that died immediately at the nil runtime check. The runtime now lives behind a gate that a watch loop keeps current: it dials while the gate is empty, and probes with a container list while it is full, clearing the gate when the probe fails so the next tick re-dials. That covers a socket that is not ready yet at boot and a containerd restarted underneath a running agent alike, and everything that used the puller or the runner reads through the gate, so a runtime arriving late is picked up without a restart. An agent with no runtime advertises no capacity. Registration and the heartbeat report zero CPU and zero memory while the gate is empty, so placement skips the host, and the control plane reads that pair of zeros as an involuntary drain: the instance becomes DRAINING with agentConnected false and zero registered resources, which is visible through DescribeContainerInstances rather than only in the node's log. Capacity returning clears it through the branch a reaped instance already used. A host with no CPU cannot exist, so the zero needs no new wire field. An operator drain is no longer cleared by a bus registration either, which it previously was. A deployment whose tasks keep failing to start now backs off. Deployment gains NextLaunchAt, and past the failure count the circuit breaker already tracks, the reconciler widens the gap between launch passes up to a cap. It is keyed on that counter but applies whether or not the breaker is enabled, because AWS leaves it off by default and an unbraked service relaunches a task that cannot start on every pass, forever.
79 lines
2.6 KiB
Go
79 lines
2.6 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/service/ecs"
|
|
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
|
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
|
)
|
|
|
|
// identity is the container instance's stable identity, assembled at boot from
|
|
// IMDS (account, instance, AZ) and config (cluster). It is the join key the
|
|
// scheduler uses to track this instance.
|
|
type identity struct {
|
|
AccountID string
|
|
ClusterName string
|
|
InstanceID string
|
|
AZ string
|
|
InstanceType string
|
|
Hostname string
|
|
Capacity bus.InstanceCapacity
|
|
AgentVersion string
|
|
}
|
|
|
|
// instanceAttributes builds the built-in container-instance attributes from the
|
|
// host's identity. Only attributes whose value was actually discovered are
|
|
// reported, so an empty one is absent rather than present and blank.
|
|
func instanceAttributes(id identity) []*ecs.Attribute {
|
|
var attrs []*ecs.Attribute
|
|
add := func(name, value string) {
|
|
if value == "" {
|
|
return
|
|
}
|
|
attrs = append(attrs, &ecs.Attribute{Name: aws.String(name), Value: aws.String(value)})
|
|
}
|
|
add(handlers_ecs.AttrInstanceType, id.InstanceType)
|
|
add(handlers_ecs.AttrAvailabilityZone, id.AZ)
|
|
return attrs
|
|
}
|
|
|
|
// advertisedIdentity is the identity to register with, given whether a
|
|
// container runtime is available. A host with no runtime can run nothing, so it
|
|
// advertises no capacity: the scheduler places nothing on it, and the control
|
|
// plane reads the zero as an involuntary drain. The identity is otherwise
|
|
// unchanged, so the instance stays visible rather than disappearing.
|
|
func advertisedIdentity(id identity, runtimeReady bool) identity {
|
|
if runtimeReady {
|
|
return id
|
|
}
|
|
id.Capacity = bus.InstanceCapacity{}
|
|
return id
|
|
}
|
|
|
|
// registrar registers the host as a container instance through the gateway.
|
|
type registrar struct {
|
|
cp controlPlane
|
|
id identity
|
|
// runtimeReady reports whether a container runtime is resolved. nil means
|
|
// always ready, which is what a test with no gate wants.
|
|
runtimeReady func() bool
|
|
}
|
|
|
|
func newRegistrar(cp controlPlane, id identity, runtimeReady func() bool) *registrar {
|
|
return ®istrar{cp: cp, id: id, runtimeReady: runtimeReady}
|
|
}
|
|
|
|
// ready reports the runtime state, defaulting to ready when no gate is wired.
|
|
func (r *registrar) ready() bool { return r.runtimeReady == nil || r.runtimeReady() }
|
|
|
|
// Register calls the gateway's RegisterContainerInstance. The scheduler records
|
|
// (or refreshes) the container instance on receipt.
|
|
func (r *registrar) Register() error {
|
|
if err := r.cp.Register(advertisedIdentity(r.id, r.ready())); err != nil {
|
|
return fmt.Errorf("register instance %s: %w", r.id.InstanceID, err)
|
|
}
|
|
return nil
|
|
}
|