mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
Fix bounded Docker update checks
Keep manual check commands active until registry collection completes. Deduplicate replayed and concurrent commands, bound collection and acknowledgement retries, surface registry result counts, and prove timeout, rate-limit, and replay behavior.
This commit is contained in:
@@ -2030,6 +2030,30 @@ last-seen, reasons, identity evidence, and non-command repair actions, and
|
||||
must never embed host-local update or uninstall commands because those can
|
||||
carry install tokens.
|
||||
|
||||
### Manual Docker update checks are bounded, replay-safe commands
|
||||
|
||||
The Docker / Podman module treats `check_updates` as one bounded agent
|
||||
operation rather than as permission to start a registry scan on every report
|
||||
poll. The monitoring-owned server queue dispatches a command only on its
|
||||
`queued` to `dispatched` transition; after receipt, the module keeps one active
|
||||
manual check plus a bounded ten-minute cache of terminal command IDs. A replay
|
||||
of the same ID may restore its current acknowledgement but must not clear the
|
||||
registry cache or collect again, and a different command arriving while the
|
||||
first is active must fail without starting another scan.
|
||||
|
||||
The first receipt clears registry result/error caches once, reports
|
||||
`in_progress`, and runs an immediate collection through the same collection
|
||||
mutex used by automatic reports. That shared mutex, not a scheduling sleep,
|
||||
serializes manual and automatic work. The manual operation inherits the
|
||||
five-minute whole-cycle ceiling and registry request cancellation from the
|
||||
Docker collector. Only after collection finishes may it report `completed`
|
||||
with checked, update, skipped, registry-error, and rate-limit counts; a fatal
|
||||
collection error or deadline reports `failed`. Terminal acknowledgement
|
||||
delivery has a small bounded retry budget, while acknowledgement failure never
|
||||
feeds the enclosing report back into command execution. The API and monitoring
|
||||
contracts continue to own host command admission, command TTL expiry, and UI
|
||||
in-flight projection.
|
||||
|
||||
### Governed action readiness remains outside agent lifecycle authority
|
||||
|
||||
The canonical Actions lifecycle may ask an executor-owned
|
||||
|
||||
+295
-57
@@ -106,44 +106,66 @@ func setAgentHeaders(req *http.Request, token string) {
|
||||
|
||||
// Agent collects Docker / Podman metrics and posts them to Pulse.
|
||||
type Agent struct {
|
||||
cfg Config
|
||||
docker dockerClient
|
||||
daemonHost string
|
||||
daemonID string // Cached at init; Podman can return unstable IDs across calls
|
||||
runtime RuntimeKind
|
||||
runtimeVer string
|
||||
agentVersion string
|
||||
supportsSwarm bool
|
||||
httpClients map[bool]*http.Client
|
||||
trustedHTTPClients map[string]*http.Client
|
||||
logger zerolog.Logger
|
||||
machineID string
|
||||
hostName string
|
||||
cpuCount int
|
||||
targets []TargetConfig
|
||||
allowedStates map[string]struct{}
|
||||
stateFilters []string
|
||||
hostID string
|
||||
prevContainerCPU map[string]cpuSample
|
||||
cpuMu sync.Mutex // protects prevContainerCPU
|
||||
reportBuffer *utils.Queue[agentsdocker.Report]
|
||||
reportBuffers map[string]*utils.Queue[agentsdocker.Report]
|
||||
registryChecker *RegistryChecker // For checking container image updates
|
||||
collectMu sync.Mutex // serializes collectOnce calls
|
||||
backgroundMu sync.Mutex // protects updateCheckRunning, cleanupTaskRunning
|
||||
updateCheckRunning bool
|
||||
cleanupTaskRunning bool
|
||||
asyncOnce sync.Once
|
||||
asyncCtx context.Context
|
||||
asyncCancel context.CancelFunc
|
||||
asyncWG sync.WaitGroup
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
cfg Config
|
||||
docker dockerClient
|
||||
daemonHost string
|
||||
daemonID string // Cached at init; Podman can return unstable IDs across calls
|
||||
runtime RuntimeKind
|
||||
runtimeVer string
|
||||
agentVersion string
|
||||
supportsSwarm bool
|
||||
httpClients map[bool]*http.Client
|
||||
trustedHTTPClients map[string]*http.Client
|
||||
logger zerolog.Logger
|
||||
machineID string
|
||||
hostName string
|
||||
cpuCount int
|
||||
targets []TargetConfig
|
||||
allowedStates map[string]struct{}
|
||||
stateFilters []string
|
||||
hostID string
|
||||
prevContainerCPU map[string]cpuSample
|
||||
cpuMu sync.Mutex // protects prevContainerCPU
|
||||
reportBuffer *utils.Queue[agentsdocker.Report]
|
||||
reportBuffers map[string]*utils.Queue[agentsdocker.Report]
|
||||
registryChecker *RegistryChecker // For checking container image updates
|
||||
collectMu sync.Mutex // serializes collectOnce calls
|
||||
manualCheckMu sync.Mutex // protects manualCheckActiveID and manualCheckResults
|
||||
manualCheckActiveID string
|
||||
manualCheckResults map[string]manualUpdateCheckResult
|
||||
manualCheckCollect func(context.Context) (agentsdocker.Report, error) // test seam for bounded manual checks
|
||||
backgroundMu sync.Mutex // protects updateCheckRunning, cleanupTaskRunning
|
||||
updateCheckRunning bool
|
||||
cleanupTaskRunning bool
|
||||
asyncOnce sync.Once
|
||||
asyncCtx context.Context
|
||||
asyncCancel context.CancelFunc
|
||||
asyncWG sync.WaitGroup
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
}
|
||||
|
||||
// ErrStopRequested indicates the agent should terminate gracefully after acknowledging a stop command.
|
||||
var ErrStopRequested = errors.New("docker host stop requested")
|
||||
|
||||
const (
|
||||
manualUpdateCheckResultTTL = 10 * time.Minute
|
||||
manualUpdateCheckResultLimit = 64
|
||||
manualUpdateCheckAckAttempts = 3
|
||||
manualUpdateCheckTerminalAckTime = 50 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
manualUpdateCheckTimeout = dockerCollectCycleTimeout
|
||||
manualUpdateCheckAckRetryDelay = 250 * time.Millisecond
|
||||
)
|
||||
|
||||
type manualUpdateCheckResult struct {
|
||||
status string
|
||||
message string
|
||||
finishedAt time.Time
|
||||
}
|
||||
|
||||
type cpuSample struct {
|
||||
totalUsage uint64
|
||||
systemUsage uint64
|
||||
@@ -807,15 +829,23 @@ func (a *Agent) waitForAsyncDelay(delay time.Duration) bool {
|
||||
}
|
||||
|
||||
func (a *Agent) collectOnce(ctx context.Context) error {
|
||||
_, err := a.collectOnceWithReport(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *Agent) collectOnceWithReport(ctx context.Context) (agentsdocker.Report, error) {
|
||||
a.collectMu.Lock()
|
||||
defer a.collectMu.Unlock()
|
||||
|
||||
report, err := a.buildReport(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build docker report: %w", err)
|
||||
return agentsdocker.Report{}, fmt.Errorf("build docker report: %w", err)
|
||||
}
|
||||
|
||||
return a.deliverReport(ctx, report)
|
||||
if err := a.deliverReport(ctx, report); err != nil {
|
||||
return report, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (a *Agent) flushBuffer(ctx context.Context) {
|
||||
@@ -1074,6 +1104,27 @@ func (a *Agent) handleCommand(ctx context.Context, target TargetConfig, command
|
||||
}
|
||||
|
||||
func (a *Agent) handleCheckUpdatesCommand(ctx context.Context, target TargetConfig, command agentsdocker.Command) error {
|
||||
command.ID = strings.TrimSpace(command.ID)
|
||||
if command.ID == "" {
|
||||
a.logger.Warn().
|
||||
Str("target", target.URL).
|
||||
Msg("Ignoring check updates command without an identifier")
|
||||
return nil
|
||||
}
|
||||
|
||||
result, shouldStart := a.beginManualUpdateCheck(command.ID)
|
||||
if !shouldStart {
|
||||
a.logger.Info().
|
||||
Str("commandID", command.ID).
|
||||
Str("target", target.URL).
|
||||
Str("status", result.status).
|
||||
Msg("Received replayed or concurrent check updates command; registry scan will not be repeated")
|
||||
if err := a.sendCommandAck(ctx, target, command.ID, result.status, result.message); err != nil {
|
||||
a.logManualUpdateCheckAckFailure(err, target, command.ID, result.status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
a.logger.Info().
|
||||
Str("commandID", command.ID).
|
||||
Str("target", target.URL).
|
||||
@@ -1083,36 +1134,223 @@ func (a *Agent) handleCheckUpdatesCommand(ctx context.Context, target TargetConf
|
||||
a.registryChecker.ForceCheck()
|
||||
}
|
||||
|
||||
// Send intermediate completion ack. Don't propagate the error — the
|
||||
// report was already delivered successfully. Propagating causes the
|
||||
// report to be buffered and retried, which re-fetches the same command
|
||||
// and creates a loop (issue #1504).
|
||||
if err := a.sendCommandAck(ctx, target, command.ID, agentsdocker.CommandStatusCompleted, "Registry cache cleared; checking for updates on next report cycle"); err != nil {
|
||||
a.logger.Warn().
|
||||
Err(err).
|
||||
Str("commandID", command.ID).
|
||||
Str("target", target.URL).
|
||||
Msg("Failed to send check updates acknowledgement")
|
||||
if err := a.sendCommandAck(ctx, target, command.ID, result.status, result.message); err != nil {
|
||||
// The server dispatches each command once. An acknowledgement failure
|
||||
// must not feed the enclosing report back into delivery/replay, and the
|
||||
// terminal acknowledgement below gets its own bounded retry budget.
|
||||
a.logManualUpdateCheckAckFailure(err, target, command.ID, result.status)
|
||||
}
|
||||
|
||||
// Trigger an immediate collection cycle to report updates.
|
||||
a.runAsync(func(asyncCtx context.Context) {
|
||||
if !a.waitForAsyncDelay(1 * time.Second) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-asyncCtx.Done():
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
_ = a.collectOnce(ctx)
|
||||
a.executeManualUpdateCheck(asyncCtx, target, command.ID)
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Agent) beginManualUpdateCheck(commandID string) (manualUpdateCheckResult, bool) {
|
||||
a.manualCheckMu.Lock()
|
||||
defer a.manualCheckMu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
if a.manualCheckResults == nil {
|
||||
a.manualCheckResults = make(map[string]manualUpdateCheckResult)
|
||||
}
|
||||
a.pruneManualUpdateCheckResultsLocked(now)
|
||||
|
||||
if result, ok := a.manualCheckResults[commandID]; ok {
|
||||
return result, false
|
||||
}
|
||||
|
||||
if a.manualCheckActiveID != "" {
|
||||
result := manualUpdateCheckResult{
|
||||
status: agentsdocker.CommandStatusFailed,
|
||||
message: "Another container update check is already running; this command was not executed",
|
||||
finishedAt: now,
|
||||
}
|
||||
a.manualCheckResults[commandID] = result
|
||||
a.trimManualUpdateCheckResultsLocked()
|
||||
return result, false
|
||||
}
|
||||
|
||||
result := manualUpdateCheckResult{
|
||||
status: agentsdocker.CommandStatusInProgress,
|
||||
message: "Checking container registries for updates",
|
||||
}
|
||||
a.manualCheckActiveID = commandID
|
||||
a.manualCheckResults[commandID] = result
|
||||
return result, true
|
||||
}
|
||||
|
||||
func (a *Agent) finishManualUpdateCheck(commandID, status, message string) manualUpdateCheckResult {
|
||||
a.manualCheckMu.Lock()
|
||||
defer a.manualCheckMu.Unlock()
|
||||
|
||||
if a.manualCheckResults == nil {
|
||||
a.manualCheckResults = make(map[string]manualUpdateCheckResult)
|
||||
}
|
||||
result := manualUpdateCheckResult{
|
||||
status: status,
|
||||
message: message,
|
||||
finishedAt: time.Now(),
|
||||
}
|
||||
a.manualCheckResults[commandID] = result
|
||||
if a.manualCheckActiveID == commandID {
|
||||
a.manualCheckActiveID = ""
|
||||
}
|
||||
a.trimManualUpdateCheckResultsLocked()
|
||||
return result
|
||||
}
|
||||
|
||||
func (a *Agent) pruneManualUpdateCheckResultsLocked(now time.Time) {
|
||||
cutoff := now.Add(-manualUpdateCheckResultTTL)
|
||||
for commandID, result := range a.manualCheckResults {
|
||||
if commandID == a.manualCheckActiveID || result.finishedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if result.finishedAt.Before(cutoff) {
|
||||
delete(a.manualCheckResults, commandID)
|
||||
}
|
||||
}
|
||||
a.trimManualUpdateCheckResultsLocked()
|
||||
}
|
||||
|
||||
func (a *Agent) trimManualUpdateCheckResultsLocked() {
|
||||
for len(a.manualCheckResults) > manualUpdateCheckResultLimit {
|
||||
var oldestID string
|
||||
var oldestFinishedAt time.Time
|
||||
for commandID, result := range a.manualCheckResults {
|
||||
if commandID == a.manualCheckActiveID || result.finishedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if oldestID == "" || result.finishedAt.Before(oldestFinishedAt) {
|
||||
oldestID = commandID
|
||||
oldestFinishedAt = result.finishedAt
|
||||
}
|
||||
}
|
||||
if oldestID == "" {
|
||||
return
|
||||
}
|
||||
delete(a.manualCheckResults, oldestID)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) executeManualUpdateCheck(asyncCtx context.Context, target TargetConfig, commandID string) {
|
||||
checkCtx, cancel := context.WithTimeout(asyncCtx, manualUpdateCheckTimeout)
|
||||
report, err := a.collectManualUpdateCheck(checkCtx)
|
||||
checkCtxErr := checkCtx.Err()
|
||||
cancel()
|
||||
|
||||
status := agentsdocker.CommandStatusCompleted
|
||||
message := summarizeManualUpdateCheck(report)
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded), errors.Is(checkCtxErr, context.DeadlineExceeded):
|
||||
status = agentsdocker.CommandStatusFailed
|
||||
message = fmt.Sprintf("Container update check timed out after %s", manualUpdateCheckTimeout)
|
||||
case errors.Is(err, context.Canceled), errors.Is(checkCtxErr, context.Canceled):
|
||||
status = agentsdocker.CommandStatusFailed
|
||||
message = "Container update check was cancelled because the agent is shutting down"
|
||||
case err != nil:
|
||||
status = agentsdocker.CommandStatusFailed
|
||||
message = fmt.Sprintf("Container update check failed: %v", err)
|
||||
}
|
||||
result := a.finishManualUpdateCheck(commandID, status, message)
|
||||
|
||||
a.logger.Info().
|
||||
Str("commandID", commandID).
|
||||
Str("target", target.URL).
|
||||
Str("status", result.status).
|
||||
Str("message", result.message).
|
||||
Msg("Container update check finished")
|
||||
|
||||
ackCtx, ackCancel := context.WithTimeout(asyncCtx, manualUpdateCheckTerminalAckTime)
|
||||
defer ackCancel()
|
||||
if err := a.sendManualUpdateCheckAckWithRetry(ackCtx, target, commandID, result); err != nil {
|
||||
a.logManualUpdateCheckAckFailure(err, target, commandID, result.status)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) collectManualUpdateCheck(ctx context.Context) (agentsdocker.Report, error) {
|
||||
if a.manualCheckCollect != nil {
|
||||
return a.manualCheckCollect(ctx)
|
||||
}
|
||||
return a.collectOnceWithReport(ctx)
|
||||
}
|
||||
|
||||
func summarizeManualUpdateCheck(report agentsdocker.Report) string {
|
||||
var checked, updates, skipped, registryErrors, rateLimited int
|
||||
for _, container := range report.Containers {
|
||||
if container.UpdateStatus == nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
updateError := strings.TrimSpace(container.UpdateStatus.Error)
|
||||
if strings.EqualFold(updateError, "digest-pinned image") {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
checked++
|
||||
if container.UpdateStatus.UpdateAvailable {
|
||||
updates++
|
||||
}
|
||||
if updateError != "" {
|
||||
registryErrors++
|
||||
if strings.Contains(strings.ToLower(updateError), "rate limit") {
|
||||
rateLimited++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
message := fmt.Sprintf("Container update check completed: %d checked, %d updates available", checked, updates)
|
||||
if skipped > 0 {
|
||||
message += fmt.Sprintf(", %d skipped", skipped)
|
||||
}
|
||||
if registryErrors > 0 {
|
||||
message += fmt.Sprintf(", %d registry errors", registryErrors)
|
||||
if rateLimited > 0 {
|
||||
message += fmt.Sprintf(" (%d rate limited)", rateLimited)
|
||||
}
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func (a *Agent) sendManualUpdateCheckAckWithRetry(ctx context.Context, target TargetConfig, commandID string, result manualUpdateCheckResult) error {
|
||||
var err error
|
||||
for attempt := 0; attempt < manualUpdateCheckAckAttempts; attempt++ {
|
||||
err = a.sendCommandAck(ctx, target, commandID, result.status, result.message)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if attempt+1 == manualUpdateCheckAckAttempts || !waitForContextDelay(ctx, manualUpdateCheckAckRetryDelay*time.Duration(1<<attempt)) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func waitForContextDelay(ctx context.Context, delay time.Duration) bool {
|
||||
if delay <= 0 {
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
timer := newTimerFn(delay)
|
||||
defer stopTimer(timer)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) logManualUpdateCheckAckFailure(err error, target TargetConfig, commandID, status string) {
|
||||
a.logger.Warn().
|
||||
Err(err).
|
||||
Str("commandID", commandID).
|
||||
Str("target", target.URL).
|
||||
Str("status", status).
|
||||
Msg("Failed to send container update check acknowledgement")
|
||||
}
|
||||
|
||||
func (a *Agent) handleStopCommand(ctx context.Context, target TargetConfig, command agentsdocker.Command) error {
|
||||
a.logger.Info().
|
||||
Str("commandID", command.ID).
|
||||
|
||||
@@ -561,6 +561,7 @@ func TestHandleCommand(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
t.Cleanup(func() { _ = agent.Close() })
|
||||
|
||||
cmd := agentsdocker.Command{
|
||||
ID: "cmd3",
|
||||
@@ -610,6 +611,9 @@ func TestHandleCommand(t *testing.T) {
|
||||
logger: zerolog.Nop(),
|
||||
hostID: "host1",
|
||||
registryChecker: registryChecker,
|
||||
manualCheckCollect: func(context.Context) (agentsdocker.Report, error) {
|
||||
return agentsdocker.Report{}, nil
|
||||
},
|
||||
}
|
||||
t.Cleanup(func() { _ = agent.Close() })
|
||||
|
||||
|
||||
@@ -2,9 +2,15 @@ package dockeragent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
|
||||
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
|
||||
@@ -69,3 +75,273 @@ func TestDockerObserverPlaintextPolicyIsDestinationScoped(t *testing.T) {
|
||||
t.Fatal("observer plaintext policy was not preserved after normalization")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualUpdateCheckRunsOnceAcrossReplayAndConcurrentRequest(t *testing.T) {
|
||||
var (
|
||||
acksMu sync.Mutex
|
||||
acks = make(map[string][]agentsdocker.CommandAck)
|
||||
)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var ack agentsdocker.CommandAck
|
||||
if err := json.NewDecoder(r.Body).Decode(&ack); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
commandID := commandIDFromAckPath(r.URL.Path)
|
||||
acksMu.Lock()
|
||||
acks[commandID] = append(acks[commandID], ack)
|
||||
acksMu.Unlock()
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var collections atomic.Int32
|
||||
registryChecker := NewRegistryChecker(zerolog.Nop())
|
||||
registryChecker.cacheDigest("before-first-command", "sha256:cached")
|
||||
agent := &Agent{
|
||||
logger: zerolog.Nop(),
|
||||
hostID: "host1",
|
||||
registryChecker: registryChecker,
|
||||
httpClients: map[bool]*http.Client{false: server.Client()},
|
||||
manualCheckCollect: func(ctx context.Context) (agentsdocker.Report, error) {
|
||||
if collections.Add(1) == 1 {
|
||||
close(started)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return agentsdocker.Report{}, ctx.Err()
|
||||
case <-release:
|
||||
return agentsdocker.Report{
|
||||
Containers: []agentsdocker.Container{{
|
||||
UpdateStatus: &agentsdocker.UpdateStatus{UpdateAvailable: true},
|
||||
}},
|
||||
}, nil
|
||||
}
|
||||
},
|
||||
}
|
||||
t.Cleanup(func() { _ = agent.Close() })
|
||||
target := TargetConfig{URL: server.URL, Token: "token"}
|
||||
first := agentsdocker.Command{ID: "check-1", Type: agentsdocker.CommandTypeCheckUpdates}
|
||||
|
||||
if err := agent.handleCheckUpdatesCommand(context.Background(), target, first); err != nil {
|
||||
t.Fatalf("start manual update check: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("manual update check did not start")
|
||||
}
|
||||
|
||||
registryChecker.cacheDigest("after-first-command", "sha256:sentinel")
|
||||
if err := agent.handleCheckUpdatesCommand(context.Background(), target, first); err != nil {
|
||||
t.Fatalf("replay manual update check: %v", err)
|
||||
}
|
||||
if err := agent.handleCheckUpdatesCommand(context.Background(), target, agentsdocker.Command{
|
||||
ID: "check-2",
|
||||
Type: agentsdocker.CommandTypeCheckUpdates,
|
||||
}); err != nil {
|
||||
t.Fatalf("send concurrent manual update check: %v", err)
|
||||
}
|
||||
|
||||
if got := collections.Load(); got != 1 {
|
||||
t.Fatalf("manual update check collections = %d, want 1", got)
|
||||
}
|
||||
if cached := registryChecker.getCached("after-first-command"); cached == nil {
|
||||
t.Fatal("replayed or concurrent command cleared the registry cache again")
|
||||
}
|
||||
waitForDockerCommandAck(t, &acksMu, acks, "check-2", func(ack agentsdocker.CommandAck) bool {
|
||||
return ack.Status == agentsdocker.CommandStatusFailed &&
|
||||
strings.Contains(ack.Message, "already running")
|
||||
})
|
||||
|
||||
close(release)
|
||||
completed := waitForDockerCommandAck(t, &acksMu, acks, "check-1", func(ack agentsdocker.CommandAck) bool {
|
||||
return ack.Status == agentsdocker.CommandStatusCompleted
|
||||
})
|
||||
if completed.Message != "Container update check completed: 1 checked, 1 updates available" {
|
||||
t.Fatalf("unexpected completion message %q", completed.Message)
|
||||
}
|
||||
|
||||
if err := agent.handleCheckUpdatesCommand(context.Background(), target, first); err != nil {
|
||||
t.Fatalf("replay completed manual update check: %v", err)
|
||||
}
|
||||
if got := collections.Load(); got != 1 {
|
||||
t.Fatalf("completed command replay collections = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualUpdateCheckTimeoutRetriesTerminalAckWithoutReexecution(t *testing.T) {
|
||||
swap(t, &manualUpdateCheckTimeout, 30*time.Millisecond)
|
||||
swap(t, &manualUpdateCheckAckRetryDelay, time.Duration(0))
|
||||
|
||||
var (
|
||||
terminalAttempts atomic.Int32
|
||||
lastTerminalMu sync.Mutex
|
||||
lastTerminal agentsdocker.CommandAck
|
||||
collections atomic.Int32
|
||||
)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var ack agentsdocker.CommandAck
|
||||
if err := json.NewDecoder(r.Body).Decode(&ack); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if ack.Status != agentsdocker.CommandStatusFailed {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
lastTerminalMu.Lock()
|
||||
lastTerminal = ack
|
||||
lastTerminalMu.Unlock()
|
||||
if terminalAttempts.Add(1) < manualUpdateCheckAckAttempts {
|
||||
http.Error(w, "temporary failure", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
agent := &Agent{
|
||||
logger: zerolog.Nop(),
|
||||
hostID: "host1",
|
||||
httpClients: map[bool]*http.Client{false: server.Client()},
|
||||
manualCheckCollect: func(ctx context.Context) (agentsdocker.Report, error) {
|
||||
collections.Add(1)
|
||||
<-ctx.Done()
|
||||
return agentsdocker.Report{}, ctx.Err()
|
||||
},
|
||||
}
|
||||
t.Cleanup(func() { _ = agent.Close() })
|
||||
target := TargetConfig{URL: server.URL, Token: "token"}
|
||||
command := agentsdocker.Command{ID: "timeout-check", Type: agentsdocker.CommandTypeCheckUpdates}
|
||||
|
||||
if err := agent.handleCheckUpdatesCommand(context.Background(), target, command); err != nil {
|
||||
t.Fatalf("start timeout check: %v", err)
|
||||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for terminalAttempts.Load() < manualUpdateCheckAckAttempts && time.Now().Before(deadline) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if got := terminalAttempts.Load(); got != manualUpdateCheckAckAttempts {
|
||||
t.Fatalf("terminal acknowledgement attempts = %d, want %d", got, manualUpdateCheckAckAttempts)
|
||||
}
|
||||
lastTerminalMu.Lock()
|
||||
terminal := lastTerminal
|
||||
lastTerminalMu.Unlock()
|
||||
if !strings.Contains(terminal.Message, "timed out after 30ms") {
|
||||
t.Fatalf("timeout acknowledgement message = %q", terminal.Message)
|
||||
}
|
||||
if got := collections.Load(); got != 1 {
|
||||
t.Fatalf("timeout command collections = %d, want 1", got)
|
||||
}
|
||||
|
||||
if err := agent.handleCheckUpdatesCommand(context.Background(), target, command); err != nil {
|
||||
t.Fatalf("replay timed-out check: %v", err)
|
||||
}
|
||||
if got := collections.Load(); got != 1 {
|
||||
t.Fatalf("timed-out command replay collections = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualUpdateCheckReportsRegistryErrorsAndRateLimits(t *testing.T) {
|
||||
var (
|
||||
acksMu sync.Mutex
|
||||
acks = make(map[string][]agentsdocker.CommandAck)
|
||||
)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var ack agentsdocker.CommandAck
|
||||
if err := json.NewDecoder(r.Body).Decode(&ack); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
acksMu.Lock()
|
||||
acks["summary-check"] = append(acks["summary-check"], ack)
|
||||
acksMu.Unlock()
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
agent := &Agent{
|
||||
logger: zerolog.Nop(),
|
||||
hostID: "host1",
|
||||
httpClients: map[bool]*http.Client{false: server.Client()},
|
||||
manualCheckCollect: func(context.Context) (agentsdocker.Report, error) {
|
||||
return agentsdocker.Report{Containers: []agentsdocker.Container{
|
||||
{UpdateStatus: &agentsdocker.UpdateStatus{UpdateAvailable: true}},
|
||||
{UpdateStatus: &agentsdocker.UpdateStatus{Error: "rate limited"}},
|
||||
{UpdateStatus: &agentsdocker.UpdateStatus{Error: "authentication required"}},
|
||||
{UpdateStatus: &agentsdocker.UpdateStatus{Error: "digest-pinned image"}},
|
||||
{},
|
||||
}}, nil
|
||||
},
|
||||
}
|
||||
t.Cleanup(func() { _ = agent.Close() })
|
||||
if err := agent.handleCheckUpdatesCommand(
|
||||
context.Background(),
|
||||
TargetConfig{URL: server.URL, Token: "token"},
|
||||
agentsdocker.Command{ID: "summary-check", Type: agentsdocker.CommandTypeCheckUpdates},
|
||||
); err != nil {
|
||||
t.Fatalf("start summary check: %v", err)
|
||||
}
|
||||
|
||||
completed := waitForDockerCommandAck(t, &acksMu, acks, "summary-check", func(ack agentsdocker.CommandAck) bool {
|
||||
return ack.Status == agentsdocker.CommandStatusCompleted
|
||||
})
|
||||
want := "Container update check completed: 3 checked, 1 updates available, 2 skipped, 2 registry errors (1 rate limited)"
|
||||
if completed.Message != want {
|
||||
t.Fatalf("completion message = %q, want %q", completed.Message, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRequestHonorsManualCheckDeadline(t *testing.T) {
|
||||
checker := &RegistryChecker{
|
||||
httpClient: &http.Client{
|
||||
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
<-req.Context().Done()
|
||||
return nil, req.Context().Err()
|
||||
}),
|
||||
},
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
startedAt := time.Now()
|
||||
_, _, err := checker.fetchDigest(ctx, "example.test", "repo", "tag", "", "", "")
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("hanging registry request error = %v, want context deadline exceeded", err)
|
||||
}
|
||||
if elapsed := time.Since(startedAt); elapsed > 500*time.Millisecond {
|
||||
t.Fatalf("hanging registry request took %s after its deadline", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func commandIDFromAckPath(path string) string {
|
||||
path = strings.TrimSuffix(path, "/ack")
|
||||
return path[strings.LastIndex(path, "/")+1:]
|
||||
}
|
||||
|
||||
func waitForDockerCommandAck(
|
||||
t *testing.T,
|
||||
mu *sync.Mutex,
|
||||
acks map[string][]agentsdocker.CommandAck,
|
||||
commandID string,
|
||||
matches func(agentsdocker.CommandAck) bool,
|
||||
) agentsdocker.CommandAck {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
mu.Lock()
|
||||
commandAcks := append([]agentsdocker.CommandAck(nil), acks[commandID]...)
|
||||
mu.Unlock()
|
||||
for _, ack := range commandAcks {
|
||||
if matches(ack) {
|
||||
return ack
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out waiting for matching acknowledgement for %q", commandID)
|
||||
return agentsdocker.CommandAck{}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user