mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-22 12:16:52 +00:00
Josh Siviour
a74052fd6f
The agent resolved the container runtime once, in New. A containerd socket that was not accepting during the first second of boot left the agent runtime-less for the life of the process, and nothing retried. It registered and heartbeated normally, so the scheduler saw a healthy instance and kept placing tasks that died immediately at the nil runtime check. The runtime now lives behind a gate that a watch loop keeps current: it dials while the gate is empty, and probes with a container list while it is full, clearing the gate when the probe fails so the next tick re-dials. That covers a socket that is not ready yet at boot and a containerd restarted underneath a running agent alike, and everything that used the puller or the runner reads through the gate, so a runtime arriving late is picked up without a restart. An agent with no runtime advertises no capacity. Registration and the heartbeat report zero CPU and zero memory while the gate is empty, so placement skips the host, and the control plane reads that pair of zeros as an involuntary drain: the instance becomes DRAINING with agentConnected false and zero registered resources, which is visible through DescribeContainerInstances rather than only in the node's log. Capacity returning clears it through the branch a reaped instance already used. A host with no CPU cannot exist, so the zero needs no new wire field. An operator drain is no longer cleared by a bus registration either, which it previously was. A deployment whose tasks keep failing to start now backs off. Deployment gains NextLaunchAt, and past the failure count the circuit breaker already tracks, the reconciler widens the gap between launch passes up to a cap. It is keyed on that counter but applies whether or not the breaker is enabled, because AWS leaves it off by default and an unbraked service relaunches a task that cannot start on every pass, forever.
97 lines
2.8 KiB
Go
97 lines
2.8 KiB
Go
package main
|
|
|
|
import (
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/mulgadc/spinifex/internal/guestenv"
|
|
)
|
|
|
|
const (
|
|
defaultEnvFile = "/etc/spinifex-ecs/agent.env"
|
|
defaultGatewayCA = "/etc/spinifex-ecs/gateway-ca.pem"
|
|
defaultIMDSBase = "http://169.254.169.254/latest"
|
|
defaultContainerdSocket = "/run/containerd/containerd.sock"
|
|
defaultHeartbeat = 30 * time.Second
|
|
defaultPollInterval = 5 * time.Second
|
|
)
|
|
|
|
// config holds the static settings the agent reads at boot. Cluster identity
|
|
// (account ID, instance ID) is discovered at runtime from IMDS, not configured;
|
|
// only the cluster *name* the instance was launched into is static. AccessKey /
|
|
// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
|
|
// gateway calls with instance-role credentials from IMDS; the agent never holds
|
|
// a NATS token.
|
|
type config struct {
|
|
GatewayURL string
|
|
GatewayCA string
|
|
Region string
|
|
ClusterName string
|
|
AccessKey string
|
|
SecretKey string
|
|
IMDSBase string
|
|
ContainerdSocket string
|
|
Heartbeat time.Duration
|
|
PollInterval time.Duration
|
|
RuntimeRetry time.Duration
|
|
CredEndpointIP string
|
|
CredEndpointPort int
|
|
}
|
|
|
|
// loadConfig reads the cloud-init env file then lets real env vars override.
|
|
func loadConfig(envFile string) config {
|
|
get := guestenv.Load(envFile).Get
|
|
|
|
cfg := config{
|
|
GatewayURL: get("ECS_GATEWAY_URL"),
|
|
GatewayCA: get("ECS_GATEWAY_CA"),
|
|
Region: get("ECS_REGION"),
|
|
ClusterName: get("ECS_CLUSTER"),
|
|
AccessKey: get("ECS_ACCESS_KEY"),
|
|
SecretKey: get("ECS_SECRET_KEY"),
|
|
IMDSBase: get("ECS_IMDS_BASE"),
|
|
ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
|
|
}
|
|
if cfg.GatewayCA == "" {
|
|
cfg.GatewayCA = defaultGatewayCA
|
|
}
|
|
if cfg.IMDSBase == "" {
|
|
cfg.IMDSBase = defaultIMDSBase
|
|
}
|
|
if cfg.ContainerdSocket == "" {
|
|
cfg.ContainerdSocket = defaultContainerdSocket
|
|
}
|
|
if cfg.ClusterName == "" {
|
|
cfg.ClusterName = "default"
|
|
}
|
|
cfg.Heartbeat = defaultHeartbeat
|
|
if v := get("ECS_HEARTBEAT_INTERVAL"); v != "" {
|
|
if d, err := time.ParseDuration(v); err == nil && d > 0 {
|
|
cfg.Heartbeat = d
|
|
}
|
|
}
|
|
cfg.PollInterval = defaultPollInterval
|
|
if v := get("ECS_POLL_INTERVAL"); v != "" {
|
|
if d, err := time.ParseDuration(v); err == nil && d > 0 {
|
|
cfg.PollInterval = d
|
|
}
|
|
}
|
|
cfg.RuntimeRetry = defaultRuntimeRetry
|
|
if v := get("ECS_RUNTIME_RETRY"); v != "" {
|
|
if d, err := time.ParseDuration(v); err == nil && d > 0 {
|
|
cfg.RuntimeRetry = d
|
|
}
|
|
}
|
|
cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
|
|
if cfg.CredEndpointIP == "" {
|
|
cfg.CredEndpointIP = defaultCredEndpointIP
|
|
}
|
|
cfg.CredEndpointPort = defaultCredEndpointPort
|
|
if v := get("ECS_CRED_ENDPOINT_PORT"); v != "" {
|
|
if p, err := strconv.Atoi(v); err == nil && p > 0 {
|
|
cfg.CredEndpointPort = p
|
|
}
|
|
}
|
|
return cfg
|
|
}
|