mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 10:35:51 +00:00
rcourtman
3086 lines
111 KiB
Go
3086 lines
111 KiB
Go
package agentexec
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/gorilla/websocket"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
var upgrader = websocket.Upgrader{
|
|
ReadBufferSize: 1024,
|
|
WriteBufferSize: 1024,
|
|
CheckOrigin: func(r *http.Request) bool {
|
|
return isAllowedWebSocketOrigin(r)
|
|
},
|
|
}
|
|
|
|
var (
|
|
jsonMarshal = json.Marshal
|
|
writeTextMessage = func(conn *websocket.Conn, data []byte) error {
|
|
return conn.WriteMessage(websocket.TextMessage, data)
|
|
}
|
|
defaultPingInterval = 5 * time.Second
|
|
pingWriteWait = 5 * time.Second
|
|
readFileTimeout = 30 * time.Second
|
|
operationQueryTimeout = 10 * time.Second
|
|
|
|
errServerShuttingDown = errors.New("agent execution server is shutting down")
|
|
)
|
|
|
|
const maxWebSocketMessageBytes int64 = 1 << 20 // 1 MiB
|
|
|
|
const (
|
|
maxAgentIDLength = 128
|
|
maxRequestIDLength = 128
|
|
maxExecuteCommandLength = 32 * 1024
|
|
maxTargetIDLength = 256
|
|
maxExecuteCommandTimeoutSeconds = 3600
|
|
defaultMaxWebSocketConnectionsPerIP = 128
|
|
defaultReadFileMaxBytes int64 = 1 << 20 // 1 MiB
|
|
maxReadFileMaxBytes int64 = 10 << 20 // 10 MiB
|
|
maxReadFilePathLength = 4096
|
|
)
|
|
|
|
var safeTargetIDPattern = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`)
|
|
var hostUpdateInventoryHashPattern = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`)
|
|
var hostStorageCleanupFingerprintPattern = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`)
|
|
|
|
// Server manages WebSocket connections from agents
|
|
type Server struct {
|
|
mu sync.RWMutex
|
|
agents map[string]*agentConn // organizationID + agentID -> connection
|
|
pendingActionRunners map[string]*agentConn // organizationID + agentID -> exact prepared runner transport
|
|
actionRunnerPromotionFences map[string]*ActionRunnerSessionPromotion // organizationID + agentID -> activation transaction fencing dispatch/results
|
|
pendingReqs map[string]chan CommandResultPayload // scoped request key -> response channel
|
|
pendingHostStorageCleanups map[string]chan HostStorageCleanupResultPayload // scoped request key -> typed storage-cleanup response
|
|
pendingHostUpdates map[string]chan HostUpdateResultPayload // scoped request key -> typed host-update response
|
|
pendingProxmoxGuestLifecycles map[string]chan ProxmoxGuestLifecycleResultPayload
|
|
pendingDockerContainerLifecycles map[string]chan DockerContainerLifecycleResultPayload
|
|
pendingDockerContainerUpdates map[string]chan DockerContainerUpdateResultPayload
|
|
pendingDockerContainerObservations map[string]chan DockerContainerObservationResultPayload
|
|
pendingActionPreflights map[string]chan ActionPreflightResultPayload
|
|
pendingHostOperations map[string]pendingHostOperation // scoped request key -> exact typed APT operation/query identity
|
|
pendingOperationQueries map[string]pendingOperationQuery
|
|
deploySubs map[string]chan DeployProgressPayload // deploySubKey(agentID, jobID) -> progress subscriber
|
|
admitToken AgentRegistrationValidator
|
|
validateSession AgentSessionValidator
|
|
commandPolicy *CommandPolicy
|
|
ipConnCounts map[string]int
|
|
maxConnsPerIP int
|
|
shutdown chan struct{}
|
|
shutdownOnce sync.Once
|
|
pingInterval time.Duration
|
|
commandAuthorizationVerifier func(CommandAuthorizationRequest) error
|
|
newCommandApprovalGrant func([]byte, string, ExecuteCommandPayload, time.Time, time.Duration) (*CommandApprovalGrant, error)
|
|
now func() time.Time
|
|
agentRegisteredNotifier func(AgentAdmission)
|
|
actionRunnerAdmissionTombstones map[string]time.Time
|
|
}
|
|
|
|
const defaultOrganizationID = "default"
|
|
|
|
type organizationContextKey struct{}
|
|
|
|
// AgentAdmission is the immutable server-owned identity of an admitted command
|
|
// session. The raw bearer token is deliberately not retained after
|
|
// registration.
|
|
type AgentAdmission struct {
|
|
OrganizationID string
|
|
TokenID string
|
|
AgentID string
|
|
Hostname string
|
|
RuntimeRole string
|
|
ActionCapability string
|
|
ActivationPending bool
|
|
}
|
|
|
|
// AgentRegistrationValidator authenticates and binds a registration to one
|
|
// organization, token, agent identity, and hostname.
|
|
type AgentRegistrationValidator func(token string, agentID string, hostname string) (AgentAdmission, bool)
|
|
|
|
// AgentSessionValidator revalidates the non-secret admission immediately
|
|
// before the server treats a socket as connected or dispatches work to it.
|
|
type AgentSessionValidator func(AgentAdmission) bool
|
|
|
|
// CommandAuthorizationRequest is the complete server-side approval scope
|
|
// verified and consumed immediately before an approval grant is signed.
|
|
type CommandAuthorizationRequest struct {
|
|
ApprovalID string
|
|
OrgID string
|
|
ActionID string
|
|
AgentID string
|
|
Command string
|
|
TargetType string
|
|
TargetID string
|
|
}
|
|
|
|
type agentConn struct {
|
|
conn *websocket.Conn
|
|
agent ConnectedAgent
|
|
admission AgentAdmission
|
|
sessionKey string
|
|
authorityKey string
|
|
approvalGrantKey []byte
|
|
writeMu sync.Mutex
|
|
done chan struct{}
|
|
doneOnce sync.Once
|
|
}
|
|
|
|
type pendingHostOperation struct {
|
|
actionID string
|
|
operation string
|
|
identity operationreceipt.Identity
|
|
subjectID string
|
|
}
|
|
|
|
type pendingOperationQuery struct {
|
|
identity operationreceipt.Identity
|
|
ch chan operationreceipt.QueryResult
|
|
}
|
|
|
|
// ActionRunnerSessionPromotion is a reversible in-memory activation
|
|
// transaction. Begin fences both dispatch and inbound-result authority before
|
|
// credential persistence; Commit swaps the exact prepared transport into the
|
|
// active map, while Rollback restores predecessor authority after a failed
|
|
// durable write. Socket cleanup is deliberately deferred until Cleanup runs
|
|
// after config.Mu has been released.
|
|
type ActionRunnerSessionPromotion struct {
|
|
server *Server
|
|
key string
|
|
pending *agentConn
|
|
cleanup []*agentConn
|
|
}
|
|
|
|
func (ac *agentConn) signalDone() {
|
|
ac.doneOnce.Do(func() {
|
|
defer func() {
|
|
// Some call sites/tests may have already closed done directly.
|
|
_ = recover()
|
|
}()
|
|
close(ac.done)
|
|
})
|
|
}
|
|
|
|
// NewServer creates a new agent execution server.
|
|
//
|
|
// validateToken is invoked during WebSocket agent registration with the token,
|
|
// the agent-claimed agentID, and the hostname from the register payload. The
|
|
// hostname is provided because enrollment-minted tokens bind to bound_hostname
|
|
// rather than to a predictable agent ID: agents derive their runtime agentID
|
|
// from /etc/machine-id (or an override), which the server cannot know when it
|
|
// mints the token. Matching on hostname preserves the trust boundary ("the
|
|
// bearer is running on the bound host") without requiring the agent to know a
|
|
// server-canonical ID format. See internal/api/router.go for the production
|
|
// validator.
|
|
func NewServer(validateToken func(token string, agentID string, hostname string) bool) *Server {
|
|
if validateToken == nil {
|
|
panic("agentexec: validateToken is required")
|
|
}
|
|
|
|
return NewServerWithAdmissionValidator(func(token string, agentID string, hostname string) (AgentAdmission, bool) {
|
|
if !validateToken(token, agentID, hostname) {
|
|
return AgentAdmission{}, false
|
|
}
|
|
return AgentAdmission{
|
|
OrganizationID: defaultOrganizationID,
|
|
AgentID: strings.TrimSpace(agentID),
|
|
Hostname: strings.TrimSpace(hostname),
|
|
}, true
|
|
}, nil)
|
|
}
|
|
|
|
// NewServerWithAdmissionValidator creates a command server whose sessions are
|
|
// tenant-scoped and can be invalidated after registration without retaining
|
|
// bearer tokens in memory.
|
|
func NewServerWithAdmissionValidator(admit AgentRegistrationValidator, validateSession AgentSessionValidator) *Server {
|
|
if admit == nil {
|
|
panic("agentexec: admission validator is required")
|
|
}
|
|
|
|
return &Server{
|
|
agents: make(map[string]*agentConn),
|
|
pendingActionRunners: make(map[string]*agentConn),
|
|
actionRunnerPromotionFences: make(map[string]*ActionRunnerSessionPromotion),
|
|
actionRunnerAdmissionTombstones: make(map[string]time.Time),
|
|
pendingReqs: make(map[string]chan CommandResultPayload),
|
|
pendingHostStorageCleanups: make(map[string]chan HostStorageCleanupResultPayload),
|
|
pendingHostUpdates: make(map[string]chan HostUpdateResultPayload),
|
|
pendingProxmoxGuestLifecycles: make(map[string]chan ProxmoxGuestLifecycleResultPayload),
|
|
pendingDockerContainerLifecycles: make(map[string]chan DockerContainerLifecycleResultPayload),
|
|
pendingDockerContainerUpdates: make(map[string]chan DockerContainerUpdateResultPayload),
|
|
pendingDockerContainerObservations: make(map[string]chan DockerContainerObservationResultPayload),
|
|
pendingActionPreflights: make(map[string]chan ActionPreflightResultPayload),
|
|
pendingHostOperations: make(map[string]pendingHostOperation),
|
|
pendingOperationQueries: make(map[string]pendingOperationQuery),
|
|
deploySubs: make(map[string]chan DeployProgressPayload),
|
|
admitToken: admit,
|
|
validateSession: validateSession,
|
|
commandPolicy: DefaultPolicy(),
|
|
ipConnCounts: make(map[string]int),
|
|
maxConnsPerIP: defaultMaxWebSocketConnectionsPerIP,
|
|
shutdown: make(chan struct{}),
|
|
pingInterval: defaultPingInterval,
|
|
newCommandApprovalGrant: NewCommandApprovalGrant,
|
|
now: time.Now,
|
|
}
|
|
}
|
|
|
|
func actionRunnerAdmissionTombstoneKey(admission AgentAdmission) string {
|
|
return strings.Join([]string{
|
|
normalizeOrganizationID(admission.OrganizationID),
|
|
strings.TrimSpace(admission.TokenID),
|
|
strings.TrimSpace(admission.AgentID),
|
|
unifiedresources.NormalizeFullHostname(admission.Hostname),
|
|
strings.TrimSpace(admission.RuntimeRole),
|
|
strings.TrimSpace(admission.ActionCapability),
|
|
}, "\x00")
|
|
}
|
|
|
|
// TombstoneActionRunnerAdmission prevents an already-admitted prepared socket
|
|
// from registering after its credential has been durably cancelled. The
|
|
// tombstone is exact and bounded to the preparation window.
|
|
func (s *Server) TombstoneActionRunnerAdmission(admission AgentAdmission, until time.Time) bool {
|
|
if s == nil || strings.TrimSpace(admission.TokenID) == "" || strings.TrimSpace(admission.AgentID) == "" ||
|
|
strings.TrimSpace(admission.Hostname) == "" ||
|
|
strings.TrimSpace(admission.RuntimeRole) != RuntimeRoleActionRunner ||
|
|
strings.TrimSpace(admission.ActionCapability) != ActionCapabilityTypedV1 {
|
|
return false
|
|
}
|
|
now := time.Now()
|
|
if s.now != nil {
|
|
now = s.now()
|
|
}
|
|
maximum := now.Add(10 * time.Minute)
|
|
if !until.After(now) || until.After(maximum) {
|
|
until = maximum
|
|
}
|
|
key := actionRunnerAdmissionTombstoneKey(admission)
|
|
s.mu.Lock()
|
|
if s.actionRunnerAdmissionTombstones == nil {
|
|
s.actionRunnerAdmissionTombstones = make(map[string]time.Time)
|
|
}
|
|
for existingKey, expiry := range s.actionRunnerAdmissionTombstones {
|
|
if !expiry.After(now) {
|
|
delete(s.actionRunnerAdmissionTombstones, existingKey)
|
|
}
|
|
}
|
|
s.actionRunnerAdmissionTombstones[key] = until
|
|
sessionKey := agentSessionKey(admission.OrganizationID, admission.AgentID)
|
|
var invalidated *agentConn
|
|
if existing, ok := s.pendingActionRunners[sessionKey]; ok && actionRunnerAdmissionTombstoneKey(existing.admission) == key {
|
|
delete(s.pendingActionRunners, sessionKey)
|
|
invalidated = existing
|
|
}
|
|
s.mu.Unlock()
|
|
if invalidated != nil {
|
|
invalidated.signalDone()
|
|
_ = invalidated.conn.Close()
|
|
}
|
|
return true
|
|
}
|
|
|
|
// WithOrganizationID scopes command-session lookup and dispatch to a tenant.
|
|
// Empty values normalize to the single-tenant default for compatibility.
|
|
func WithOrganizationID(ctx context.Context, organizationID string) context.Context {
|
|
if ctx == nil {
|
|
ctx = context.Background()
|
|
}
|
|
return context.WithValue(ctx, organizationContextKey{}, normalizeOrganizationID(organizationID))
|
|
}
|
|
|
|
func normalizeOrganizationID(organizationID string) string {
|
|
if organizationID = strings.TrimSpace(organizationID); organizationID != "" {
|
|
return organizationID
|
|
}
|
|
return defaultOrganizationID
|
|
}
|
|
|
|
func organizationIDFromContext(ctx context.Context) string {
|
|
if ctx != nil {
|
|
if organizationID, ok := ctx.Value(organizationContextKey{}).(string); ok {
|
|
return normalizeOrganizationID(organizationID)
|
|
}
|
|
}
|
|
return defaultOrganizationID
|
|
}
|
|
|
|
// OrganizationServer is a tenant-pinned view of a command server. It exists
|
|
// for consumers whose interface cannot carry the request context through
|
|
// discovery and dispatch as separate calls (for example, long-lived per-tenant
|
|
// Assistant services).
|
|
type OrganizationServer struct {
|
|
server *Server
|
|
organizationID string
|
|
}
|
|
|
|
// ForOrganization returns a command-server view that can only discover and
|
|
// dispatch sessions admitted to organizationID.
|
|
func (s *Server) ForOrganization(organizationID string) *OrganizationServer {
|
|
return &OrganizationServer{
|
|
server: s,
|
|
organizationID: normalizeOrganizationID(organizationID),
|
|
}
|
|
}
|
|
|
|
func (s *OrganizationServer) GetConnectedAgents() []ConnectedAgent {
|
|
if s == nil || s.server == nil {
|
|
return nil
|
|
}
|
|
return s.server.GetConnectedAgentsForOrganization(s.organizationID)
|
|
}
|
|
|
|
func (s *OrganizationServer) ExecuteCommand(ctx context.Context, agentID string, cmd ExecuteCommandPayload) (*CommandResultPayload, error) {
|
|
if s == nil || s.server == nil {
|
|
return nil, fmt.Errorf("agent execution server is unavailable")
|
|
}
|
|
return s.server.ExecuteCommand(WithOrganizationID(ctx, s.organizationID), agentID, cmd)
|
|
}
|
|
|
|
func agentSessionKey(organizationID, agentID string) string {
|
|
organizationID = normalizeOrganizationID(organizationID)
|
|
agentID = strings.TrimSpace(agentID)
|
|
if organizationID == defaultOrganizationID {
|
|
// Preserve the historical key for direct single-tenant users and tests.
|
|
return agentID
|
|
}
|
|
return organizationID + "\x00" + agentID
|
|
}
|
|
|
|
func connectionSessionKey(ac *agentConn) string {
|
|
if ac == nil {
|
|
return ""
|
|
}
|
|
if strings.TrimSpace(ac.sessionKey) != "" {
|
|
return ac.sessionKey
|
|
}
|
|
return agentSessionKey(ac.admission.OrganizationID, ac.agent.AgentID)
|
|
}
|
|
|
|
// connectionAuthorityKey identifies one admitted WebSocket generation. Host
|
|
// identity is intentionally insufficient for request correlation: an action-
|
|
// runner replacement must not inherit work dispatched to its predecessor.
|
|
func connectionAuthorityKey(ac *agentConn) string {
|
|
if ac == nil {
|
|
return ""
|
|
}
|
|
sessionKey := connectionSessionKey(ac)
|
|
if strings.TrimSpace(ac.authorityKey) == "" {
|
|
return sessionKey
|
|
}
|
|
return sessionKey + "\x00" + ac.authorityKey
|
|
}
|
|
|
|
// activeConnectionLocked reports whether ac is the exact transport currently
|
|
// authorized to satisfy inbound work for its tenant and host. The caller must
|
|
// hold s.mu for reading or writing across this check and the corresponding
|
|
// channel delivery so action-runner promotion cannot create a check/send race.
|
|
func (s *Server) activeConnectionLocked(ac *agentConn) bool {
|
|
if s == nil || ac == nil {
|
|
return false
|
|
}
|
|
key := connectionSessionKey(ac)
|
|
return s.actionRunnerPromotionFences[key] == nil && s.agents[key] == ac
|
|
}
|
|
|
|
// SetCommandAuthorizationVerifier installs the server-owned authorization
|
|
// consumer used for approval-gated arbitrary commands.
|
|
func (s *Server) SetCommandAuthorizationVerifier(verifier func(CommandAuthorizationRequest) error) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
s.commandAuthorizationVerifier = verifier
|
|
}
|
|
|
|
// SetAgentRegisteredNotifier installs a callback fired after an agent
|
|
// completes registration (including a reconnect that replaces an existing
|
|
// connection). Durable-dispatch recovery hangs off this: receipt-pending
|
|
// reconciliation can only query an agent while it is connected, so the
|
|
// registration itself is the recovery trigger. The callback runs on its own
|
|
// goroutine because the query response can only be read once this server
|
|
// enters the connection's read loop.
|
|
func (s *Server) SetAgentRegisteredNotifier(notify func(agentID string)) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
if notify == nil {
|
|
s.agentRegisteredNotifier = nil
|
|
return
|
|
}
|
|
s.agentRegisteredNotifier = func(admission AgentAdmission) {
|
|
notify(admission.AgentID)
|
|
}
|
|
}
|
|
|
|
// SetAgentAdmissionNotifier installs the tenant-aware registration callback.
|
|
func (s *Server) SetAgentAdmissionNotifier(notify func(AgentAdmission)) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
s.agentRegisteredNotifier = notify
|
|
}
|
|
|
|
func (s *Server) isShuttingDown() bool {
|
|
select {
|
|
case <-s.shutdown:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func pendingRequestKey(authorityKey, requestID string) string {
|
|
return authorityKey + "\x00" + requestID
|
|
}
|
|
|
|
func (s *Server) connectionForOrganization(organizationID, agentID string) (*agentConn, bool) {
|
|
if s == nil {
|
|
return nil, false
|
|
}
|
|
key := agentSessionKey(organizationID, agentID)
|
|
s.mu.RLock()
|
|
ac, ok := s.agents[key]
|
|
fenced := s.actionRunnerPromotionFences[key] != nil
|
|
s.mu.RUnlock()
|
|
if !ok || fenced {
|
|
return nil, false
|
|
}
|
|
// Prepared action runners live only in pendingActionRunners. Membership in
|
|
// the active map is the immutable dispatch-authority decision; do not read
|
|
// or mutate admission state outside the server lock during promotion.
|
|
if s.validateSession == nil || s.validateSession(ac.admission) {
|
|
return ac, true
|
|
}
|
|
|
|
// A revoked, expired, re-bound, or otherwise stale token must stop being a
|
|
// command authority immediately. Pointer equality prevents an old
|
|
// validation result from evicting a replacement session.
|
|
s.mu.Lock()
|
|
if current, exists := s.agents[key]; exists && current == ac {
|
|
delete(s.agents, key)
|
|
}
|
|
s.mu.Unlock()
|
|
ac.signalDone()
|
|
if ac.conn != nil {
|
|
_ = ac.conn.Close()
|
|
}
|
|
return nil, false
|
|
}
|
|
|
|
// HasActionRunnerSession reports whether the exact runner transport is
|
|
// currently registered. It intentionally bypasses dispatch readiness: the
|
|
// activation endpoint uses this proof to promote a pending session.
|
|
func (s *Server) HasActionRunnerSession(admission AgentAdmission) bool {
|
|
if s == nil {
|
|
return false
|
|
}
|
|
key := agentSessionKey(admission.OrganizationID, admission.AgentID)
|
|
s.mu.RLock()
|
|
current, ok := s.pendingActionRunners[key]
|
|
s.mu.RUnlock()
|
|
return ok && current != nil && admission.ActivationPending && sameActionRunnerAdmission(current.admission, admission)
|
|
}
|
|
|
|
// BeginActionRunnerSessionPromotion fences the host's active map entry before
|
|
// the credential inventory can be durably changed. The fence makes both new
|
|
// dispatch and inbound-result delivery fail closed while persistence is in
|
|
// progress. Callers must resolve the returned transaction with Commit,
|
|
// Rollback, or FailClosed.
|
|
func (s *Server) BeginActionRunnerSessionPromotion(admission AgentAdmission) (*ActionRunnerSessionPromotion, bool) {
|
|
if s == nil {
|
|
return nil, false
|
|
}
|
|
key := agentSessionKey(admission.OrganizationID, admission.AgentID)
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.actionRunnerPromotionFences[key] != nil {
|
|
return nil, false
|
|
}
|
|
pending, ok := s.pendingActionRunners[key]
|
|
if !ok || pending == nil || !admission.ActivationPending || !sameActionRunnerAdmission(pending.admission, admission) {
|
|
return nil, false
|
|
}
|
|
tx := &ActionRunnerSessionPromotion{server: s, key: key, pending: pending}
|
|
s.actionRunnerPromotionFences[key] = tx
|
|
return tx, true
|
|
}
|
|
|
|
// Commit atomically promotes the exact transport captured by Begin. It
|
|
// returns false if that prepared socket disconnected or was replaced while
|
|
// persistence was in progress; the fence remains in place until Rollback or
|
|
// FailClosed resolves the transaction.
|
|
func (tx *ActionRunnerSessionPromotion) Commit() bool {
|
|
if tx == nil || tx.server == nil {
|
|
return false
|
|
}
|
|
s := tx.server
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.actionRunnerPromotionFences[tx.key] != tx || s.pendingActionRunners[tx.key] != tx.pending {
|
|
return false
|
|
}
|
|
delete(s.pendingActionRunners, tx.key)
|
|
displaced := s.agents[tx.key]
|
|
s.agents[tx.key] = tx.pending
|
|
delete(s.actionRunnerPromotionFences, tx.key)
|
|
if displaced != nil && displaced != tx.pending {
|
|
tx.cleanup = append(tx.cleanup, displaced)
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Rollback removes this transaction's fence without changing either session
|
|
// map. It is valid only after the prior credential inventory is known durable.
|
|
func (tx *ActionRunnerSessionPromotion) Rollback() {
|
|
if tx == nil || tx.server == nil {
|
|
return
|
|
}
|
|
s := tx.server
|
|
s.mu.Lock()
|
|
if s.actionRunnerPromotionFences[tx.key] != tx {
|
|
s.mu.Unlock()
|
|
return
|
|
}
|
|
delete(s.actionRunnerPromotionFences, tx.key)
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
// FailClosed resolves an indeterminate durable activation by removing every
|
|
// active or prepared transport for this host. Neither the potentially revoked
|
|
// predecessor nor an uncommitted replacement may retain runtime authority.
|
|
func (tx *ActionRunnerSessionPromotion) FailClosed() {
|
|
if tx == nil || tx.server == nil {
|
|
return
|
|
}
|
|
s := tx.server
|
|
s.mu.Lock()
|
|
if s.actionRunnerPromotionFences[tx.key] != tx {
|
|
s.mu.Unlock()
|
|
return
|
|
}
|
|
delete(s.actionRunnerPromotionFences, tx.key)
|
|
if active := s.agents[tx.key]; active != nil {
|
|
delete(s.agents, tx.key)
|
|
tx.cleanup = append(tx.cleanup, active)
|
|
}
|
|
if pending := s.pendingActionRunners[tx.key]; pending != nil {
|
|
delete(s.pendingActionRunners, tx.key)
|
|
tx.cleanup = append(tx.cleanup, pending)
|
|
}
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
// Cleanup closes transports displaced by Commit or removed by FailClosed. It
|
|
// must run after the caller releases config.Mu so socket I/O never occurs
|
|
// while either the credential or session-map transaction lock is held.
|
|
func (tx *ActionRunnerSessionPromotion) Cleanup() {
|
|
if tx == nil {
|
|
return
|
|
}
|
|
seen := make(map[*agentConn]struct{}, len(tx.cleanup))
|
|
for _, ac := range tx.cleanup {
|
|
if ac == nil {
|
|
continue
|
|
}
|
|
if _, duplicate := seen[ac]; duplicate {
|
|
continue
|
|
}
|
|
seen[ac] = struct{}{}
|
|
ac.signalDone()
|
|
if ac.conn != nil {
|
|
_ = ac.conn.Close()
|
|
}
|
|
}
|
|
}
|
|
|
|
// PromoteActionRunnerSessionForCommit is the bounded compatibility wrapper
|
|
// used by direct callers and tests. Production credential activation uses the
|
|
// full Begin/Commit/Rollback transaction so the persistence interval is fenced.
|
|
func (s *Server) PromoteActionRunnerSessionForCommit(admission AgentAdmission) (func(), bool) {
|
|
tx, ok := s.BeginActionRunnerSessionPromotion(admission)
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
if !tx.Commit() {
|
|
tx.Rollback()
|
|
return nil, false
|
|
}
|
|
if len(tx.cleanup) == 0 {
|
|
return nil, true
|
|
}
|
|
return tx.Cleanup, true
|
|
}
|
|
|
|
// PromoteActionRunnerSession is the non-transactional compatibility wrapper.
|
|
// Production activation uses PromoteActionRunnerSessionForCommit and defers
|
|
// cleanup until after config.Mu has been released.
|
|
func (s *Server) PromoteActionRunnerSession(admission AgentAdmission) bool {
|
|
cleanup, promoted := s.PromoteActionRunnerSessionForCommit(admission)
|
|
if cleanup != nil {
|
|
cleanup()
|
|
}
|
|
return promoted
|
|
}
|
|
|
|
// InvalidateActionRunnerSession closes exactly the currently admitted typed
|
|
// action-runner session identified by admission. A stale rotation result must
|
|
// never evict a replacement session that has already registered for the same
|
|
// tenant and host identity.
|
|
func (s *Server) InvalidateActionRunnerSession(admission AgentAdmission) bool {
|
|
if s == nil {
|
|
return false
|
|
}
|
|
expected := AgentAdmission{
|
|
OrganizationID: normalizeOrganizationID(admission.OrganizationID),
|
|
TokenID: strings.TrimSpace(admission.TokenID),
|
|
AgentID: strings.TrimSpace(admission.AgentID),
|
|
Hostname: strings.TrimSpace(admission.Hostname),
|
|
RuntimeRole: strings.TrimSpace(admission.RuntimeRole),
|
|
ActionCapability: strings.TrimSpace(admission.ActionCapability),
|
|
}
|
|
if expected.TokenID == "" || expected.AgentID == "" || expected.Hostname == "" ||
|
|
expected.RuntimeRole != RuntimeRoleActionRunner || expected.ActionCapability != ActionCapabilityTypedV1 {
|
|
return false
|
|
}
|
|
key := agentSessionKey(expected.OrganizationID, expected.AgentID)
|
|
s.mu.Lock()
|
|
var current *agentConn
|
|
if active, ok := s.agents[key]; ok && active != nil && sameActionRunnerAdmission(active.admission, expected) {
|
|
delete(s.agents, key)
|
|
current = active
|
|
} else if pending, ok := s.pendingActionRunners[key]; ok && pending != nil && sameActionRunnerAdmission(pending.admission, expected) {
|
|
delete(s.pendingActionRunners, key)
|
|
current = pending
|
|
}
|
|
s.mu.Unlock()
|
|
if current == nil {
|
|
return false
|
|
}
|
|
|
|
current.signalDone()
|
|
if current.conn != nil {
|
|
_ = current.conn.Close()
|
|
}
|
|
return true
|
|
}
|
|
|
|
// InvalidateAgentSession closes exactly the currently admitted session for a
|
|
// credential whose authority has changed. Unlike action-runner rotation this
|
|
// accepts either runtime role, but every admission field must still match so a
|
|
// stale or cross-tenant credential transition cannot evict another session.
|
|
func (s *Server) InvalidateAgentSession(admission AgentAdmission) bool {
|
|
if s == nil {
|
|
return false
|
|
}
|
|
expected := AgentAdmission{
|
|
OrganizationID: normalizeOrganizationID(admission.OrganizationID),
|
|
TokenID: strings.TrimSpace(admission.TokenID),
|
|
AgentID: strings.TrimSpace(admission.AgentID),
|
|
Hostname: strings.TrimSpace(admission.Hostname),
|
|
RuntimeRole: strings.TrimSpace(admission.RuntimeRole),
|
|
ActionCapability: strings.TrimSpace(admission.ActionCapability),
|
|
}
|
|
if expected.TokenID == "" || expected.AgentID == "" || expected.Hostname == "" || expected.RuntimeRole == "" {
|
|
return false
|
|
}
|
|
key := agentSessionKey(expected.OrganizationID, expected.AgentID)
|
|
s.mu.Lock()
|
|
current, ok := s.agents[key]
|
|
if !ok || current == nil ||
|
|
normalizeOrganizationID(current.admission.OrganizationID) != expected.OrganizationID ||
|
|
strings.TrimSpace(current.admission.TokenID) != expected.TokenID ||
|
|
strings.TrimSpace(current.admission.AgentID) != expected.AgentID ||
|
|
!unifiedresources.HostnamesEquivalent(current.admission.Hostname, expected.Hostname) ||
|
|
strings.TrimSpace(current.admission.RuntimeRole) != expected.RuntimeRole ||
|
|
strings.TrimSpace(current.admission.ActionCapability) != expected.ActionCapability {
|
|
s.mu.Unlock()
|
|
return false
|
|
}
|
|
delete(s.agents, key)
|
|
s.mu.Unlock()
|
|
current.signalDone()
|
|
if current.conn != nil {
|
|
_ = current.conn.Close()
|
|
}
|
|
return true
|
|
}
|
|
|
|
func sameActionRunnerAdmission(current, expected AgentAdmission) bool {
|
|
return normalizeOrganizationID(current.OrganizationID) == expected.OrganizationID &&
|
|
strings.TrimSpace(current.TokenID) == expected.TokenID &&
|
|
strings.TrimSpace(current.AgentID) == expected.AgentID &&
|
|
(strings.EqualFold(strings.TrimSpace(current.Hostname), expected.Hostname) ||
|
|
unifiedresources.HostnamesEquivalent(current.Hostname, expected.Hostname)) &&
|
|
strings.TrimSpace(current.RuntimeRole) == RuntimeRoleActionRunner &&
|
|
strings.TrimSpace(current.ActionCapability) == ActionCapabilityTypedV1
|
|
}
|
|
|
|
func requireLegacyFullTrustConnection(ac *agentConn, operation string) error {
|
|
if ac == nil {
|
|
return fmt.Errorf("agent connection is unavailable")
|
|
}
|
|
if strings.TrimSpace(ac.admission.RuntimeRole) == RuntimeRoleActionRunner {
|
|
return fmt.Errorf("%s is not available on typed action-runner sessions", operation)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) connectionForContext(ctx context.Context, agentID string) (*agentConn, bool) {
|
|
return s.connectionForOrganization(organizationIDFromContext(ctx), agentID)
|
|
}
|
|
|
|
func (s *Server) claimPendingHostOperation(authorityKey, requestID, actionID, operation string) (string, error) {
|
|
key := pendingRequestKey(authorityKey, requestID)
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if _, exists := s.pendingHostOperations[key]; exists {
|
|
return "", fmt.Errorf("typed host operation request %q is already pending", requestID)
|
|
}
|
|
s.pendingHostOperations[key] = pendingHostOperation{
|
|
actionID: strings.TrimSpace(actionID),
|
|
operation: strings.TrimSpace(operation),
|
|
}
|
|
return key, nil
|
|
}
|
|
|
|
func (s *Server) matchesPendingHostOperation(authorityKey, requestID, actionID, operation string) bool {
|
|
key := pendingRequestKey(authorityKey, requestID)
|
|
s.mu.RLock()
|
|
expected, ok := s.pendingHostOperations[key]
|
|
s.mu.RUnlock()
|
|
return ok && expected.actionID == strings.TrimSpace(actionID) && expected.operation == strings.TrimSpace(operation)
|
|
}
|
|
|
|
func (s *Server) claimPendingDockerOperation(identity operationreceipt.Identity, containerID string) (string, error) {
|
|
return s.claimPendingDockerOperationForSession(identity.AgentID, identity, containerID)
|
|
}
|
|
|
|
func (s *Server) claimPendingDockerOperationForSession(authorityKey string, identity operationreceipt.Identity, containerID string) (string, error) {
|
|
identity, err := operationreceipt.NormalizeIdentity(identity)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
key := pendingRequestKey(authorityKey, identity.AttemptID)
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if _, exists := s.pendingHostOperations[key]; exists {
|
|
return "", fmt.Errorf("typed operation request %q is already pending", identity.AttemptID)
|
|
}
|
|
s.pendingHostOperations[key] = pendingHostOperation{actionID: identity.ActionID, operation: identity.OperationKind, identity: identity, subjectID: strings.ToLower(strings.TrimSpace(containerID))}
|
|
return key, nil
|
|
}
|
|
|
|
func (s *Server) matchesPendingDockerOperation(agentID string, result DockerContainerLifecycleResultPayload) bool {
|
|
return s.matchesPendingDockerOperationForSession(agentID, agentID, result)
|
|
}
|
|
|
|
func (s *Server) matchesPendingDockerOperationForSession(authorityKey, agentID string, result DockerContainerLifecycleResultPayload) bool {
|
|
key := pendingRequestKey(authorityKey, result.RequestID)
|
|
s.mu.RLock()
|
|
expected, ok := s.pendingHostOperations[key]
|
|
s.mu.RUnlock()
|
|
actual := operationreceipt.Identity{AttemptID: result.RequestID, ActionID: result.ActionID, OperationKind: result.Operation, OperationVersion: result.OperationVersion, RequestDigest: result.RequestDigest, AgentID: strings.TrimSpace(agentID)}
|
|
return ok && expected.identity == actual && expected.subjectID == strings.ToLower(strings.TrimSpace(result.ContainerID))
|
|
}
|
|
|
|
func (s *Server) matchesPendingDockerUpdateOperation(agentID string, result DockerContainerUpdateResultPayload) bool {
|
|
return s.matchesPendingDockerUpdateOperationForSession(agentID, agentID, result)
|
|
}
|
|
|
|
func (s *Server) matchesPendingDockerUpdateOperationForSession(authorityKey, agentID string, result DockerContainerUpdateResultPayload) bool {
|
|
key := pendingRequestKey(authorityKey, result.RequestID)
|
|
s.mu.RLock()
|
|
expected, ok := s.pendingHostOperations[key]
|
|
s.mu.RUnlock()
|
|
actual := operationreceipt.Identity{AttemptID: result.RequestID, ActionID: result.ActionID, OperationKind: result.Operation, OperationVersion: result.OperationVersion, RequestDigest: result.RequestDigest, AgentID: strings.TrimSpace(agentID)}
|
|
return ok && expected.identity == actual && expected.subjectID == strings.ToLower(strings.TrimSpace(result.ContainerID))
|
|
}
|
|
|
|
func (s *Server) matchesPendingProxmoxGuestOperationForSession(authorityKey, agentID string, result ProxmoxGuestLifecycleResultPayload) bool {
|
|
key := pendingRequestKey(authorityKey, result.RequestID)
|
|
s.mu.RLock()
|
|
expected, ok := s.pendingHostOperations[key]
|
|
s.mu.RUnlock()
|
|
actual := operationreceipt.Identity{AttemptID: result.RequestID, ActionID: result.ActionID, OperationKind: result.Operation, OperationVersion: result.OperationVersion, RequestDigest: result.RequestDigest, AgentID: strings.TrimSpace(agentID)}
|
|
subject := result.GuestKind + ":" + strconv.Itoa(result.VMID)
|
|
return ok && expected.identity == actual && expected.subjectID == subject
|
|
}
|
|
|
|
func (s *Server) releasePendingHostOperation(key string) {
|
|
s.mu.Lock()
|
|
delete(s.pendingHostOperations, key)
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
func deploySubKey(agentID, jobID string) string {
|
|
return agentID + "\x00" + jobID
|
|
}
|
|
|
|
func normalizeTarget(targetType, targetID string) (string, string, error) {
|
|
normalizedType := strings.ToLower(strings.TrimSpace(targetType))
|
|
if normalizedType == "" {
|
|
normalizedType = "agent"
|
|
}
|
|
|
|
normalizedTargetID := strings.TrimSpace(targetID)
|
|
switch normalizedType {
|
|
case "agent":
|
|
// Agent-level execution ignores target ID.
|
|
return "agent", "", nil
|
|
case "container", "vm":
|
|
if normalizedTargetID == "" {
|
|
return "", "", fmt.Errorf("target id is required for target type %q", normalizedType)
|
|
}
|
|
if len(normalizedTargetID) > maxTargetIDLength {
|
|
return "", "", fmt.Errorf("target id exceeds %d characters", maxTargetIDLength)
|
|
}
|
|
if !safeTargetIDPattern.MatchString(normalizedTargetID) {
|
|
return "", "", fmt.Errorf("target id contains invalid characters")
|
|
}
|
|
return normalizedType, normalizedTargetID, nil
|
|
default:
|
|
return "", "", fmt.Errorf("invalid target type %q", targetType)
|
|
}
|
|
}
|
|
|
|
func validateExecuteCommandPayload(cmd *ExecuteCommandPayload) error {
|
|
if cmd == nil {
|
|
return fmt.Errorf("command payload is required")
|
|
}
|
|
|
|
if strings.TrimSpace(cmd.Command) == "" {
|
|
return fmt.Errorf("command is required")
|
|
}
|
|
cmd.ApprovalID = strings.TrimSpace(cmd.ApprovalID)
|
|
if len(cmd.Command) > maxExecuteCommandLength {
|
|
return fmt.Errorf("command exceeds %d characters", maxExecuteCommandLength)
|
|
}
|
|
|
|
targetType, targetID, err := normalizeTarget(cmd.TargetType, cmd.TargetID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cmd.TargetType = targetType
|
|
cmd.TargetID = targetID
|
|
|
|
if cmd.Timeout < 0 {
|
|
return fmt.Errorf("timeout cannot be negative")
|
|
}
|
|
if cmd.Timeout > maxExecuteCommandTimeoutSeconds {
|
|
return fmt.Errorf("timeout cannot exceed %d seconds", maxExecuteCommandTimeoutSeconds)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) authorizeCommandPayload(cmd ExecuteCommandPayload) error {
|
|
if s == nil || s.commandPolicy == nil {
|
|
return nil
|
|
}
|
|
|
|
switch s.commandPolicy.Evaluate(cmd.Command) {
|
|
case PolicyBlock:
|
|
return fmt.Errorf("command blocked by policy")
|
|
case PolicyRequireApproval:
|
|
// Trusted internal subsystems (e.g. servicediscovery deep scans) carry
|
|
// a hardcoded command catalog and never accept user-supplied commands,
|
|
// so the user-driven approval gate does not apply to them.
|
|
if cmd.Trusted {
|
|
return nil
|
|
}
|
|
if cmd.ApprovalID == "" {
|
|
return fmt.Errorf("command requires approval")
|
|
}
|
|
if cmd.authorization == nil || strings.TrimSpace(cmd.authorization.ActionID) == "" {
|
|
return fmt.Errorf("command requires server-owned approval authorization")
|
|
}
|
|
if s.commandAuthorizationVerifier == nil {
|
|
return fmt.Errorf("command approval authorization verifier is unavailable")
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateReadFilePayload(req *ReadFilePayload) error {
|
|
if req == nil {
|
|
return fmt.Errorf("read file payload is required")
|
|
}
|
|
|
|
req.Path = strings.TrimSpace(req.Path)
|
|
if req.Path == "" {
|
|
return fmt.Errorf("path is required")
|
|
}
|
|
if len(req.Path) > maxReadFilePathLength {
|
|
return fmt.Errorf("path exceeds %d characters", maxReadFilePathLength)
|
|
}
|
|
if strings.ContainsAny(req.Path, "\x00\r\n") {
|
|
return fmt.Errorf("path contains invalid control characters")
|
|
}
|
|
|
|
targetType, targetID, err := normalizeTarget(req.TargetType, req.TargetID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.TargetType = targetType
|
|
req.TargetID = targetID
|
|
|
|
if req.MaxBytes < 0 {
|
|
return fmt.Errorf("max bytes cannot be negative")
|
|
}
|
|
if req.MaxBytes == 0 {
|
|
req.MaxBytes = defaultReadFileMaxBytes
|
|
}
|
|
if req.MaxBytes > maxReadFileMaxBytes {
|
|
return fmt.Errorf("max bytes cannot exceed %d", maxReadFileMaxBytes)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateHostUpdatePayload(req *HostUpdatePayload) error {
|
|
if req == nil {
|
|
return fmt.Errorf("host update payload is required")
|
|
}
|
|
req.RequestID = strings.TrimSpace(req.RequestID)
|
|
req.ActionID = strings.TrimSpace(req.ActionID)
|
|
req.Operation = strings.TrimSpace(req.Operation)
|
|
req.ExpectedInventoryHash = strings.TrimSpace(req.ExpectedInventoryHash)
|
|
if req.RequestID == "" {
|
|
return fmt.Errorf("request id is required")
|
|
}
|
|
if len(req.RequestID) > maxRequestIDLength {
|
|
return fmt.Errorf("request id exceeds %d characters", maxRequestIDLength)
|
|
}
|
|
if req.ActionID == "" {
|
|
return fmt.Errorf("action id is required")
|
|
}
|
|
if len(req.ActionID) > maxRequestIDLength {
|
|
return fmt.Errorf("action id exceeds %d characters", maxRequestIDLength)
|
|
}
|
|
if req.Operation != HostUpdateOperationInstall {
|
|
return fmt.Errorf("unsupported host update operation %q", req.Operation)
|
|
}
|
|
if req.OperationVersion != HostAPTOperationVersion {
|
|
return fmt.Errorf("unsupported host update operation version %d", req.OperationVersion)
|
|
}
|
|
expectedDigest, err := hostUpdateRequestDigest(*req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if req.RequestDigest != expectedDigest {
|
|
return fmt.Errorf("host update request digest mismatch")
|
|
}
|
|
if !hostUpdateInventoryHashPattern.MatchString(req.ExpectedInventoryHash) {
|
|
return fmt.Errorf("expected inventory hash is required and must be sha256")
|
|
}
|
|
if req.Timeout < 0 || req.Timeout > 1800 {
|
|
return fmt.Errorf("host update timeout must be between 0 and 1800 seconds")
|
|
}
|
|
if req.Timeout == 0 {
|
|
req.Timeout = 900
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateHostUpdateResultPayload(result *HostUpdateResultPayload) error {
|
|
if result == nil {
|
|
return fmt.Errorf("host update result is required")
|
|
}
|
|
result.RequestID = strings.TrimSpace(result.RequestID)
|
|
result.Verification = strings.TrimSpace(result.Verification)
|
|
result.ReasonCode = strings.TrimSpace(result.ReasonCode)
|
|
if result.RequestID == "" || len(result.RequestID) > maxRequestIDLength {
|
|
return fmt.Errorf("invalid request id")
|
|
}
|
|
if result.Before.PendingCount < 0 || result.After.PendingCount < 0 {
|
|
return fmt.Errorf("pending package counts cannot be negative")
|
|
}
|
|
if len(result.Before.Packages) > 200 || len(result.After.Packages) > 200 {
|
|
return fmt.Errorf("package evidence exceeds bounded limit")
|
|
}
|
|
for _, hash := range []string{result.Before.InventoryHash, result.After.InventoryHash} {
|
|
if hash != "" && !hostUpdateInventoryHashPattern.MatchString(hash) {
|
|
return fmt.Errorf("invalid package inventory hash")
|
|
}
|
|
}
|
|
switch result.Verification {
|
|
case HostUpdateVerificationVerified:
|
|
if !result.Success || !result.After.Supported || result.After.Manager != "apt" || result.After.Error != "" || result.After.PendingCount != 0 || result.After.InventoryHash == "" {
|
|
return fmt.Errorf("verified host update lacks a valid zero-pending postcondition")
|
|
}
|
|
case HostUpdateVerificationFailed, HostUpdateVerificationInconclusive:
|
|
default:
|
|
return fmt.Errorf("unsupported host update verification %q", result.Verification)
|
|
}
|
|
if len(result.Error) > 1024 {
|
|
return fmt.Errorf("host update error exceeds bounded limit")
|
|
}
|
|
if result.ReasonCode != "" && !IsActionRefusalReasonCode(result.ReasonCode) {
|
|
return fmt.Errorf("invalid host update refusal reason code")
|
|
}
|
|
if result.ReasonCode != "" && (result.MutationStarted || result.Success) {
|
|
return fmt.Errorf("host update refusal reason conflicts with execution state")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateHostStorageCleanupPayload(req *HostStorageCleanupPayload) error {
|
|
if req == nil {
|
|
return fmt.Errorf("host storage cleanup payload is required")
|
|
}
|
|
req.RequestID = strings.TrimSpace(req.RequestID)
|
|
req.ActionID = strings.TrimSpace(req.ActionID)
|
|
req.Operation = strings.TrimSpace(req.Operation)
|
|
req.ExpectedFingerprint = strings.TrimSpace(req.ExpectedFingerprint)
|
|
if req.RequestID == "" || len(req.RequestID) > maxRequestIDLength {
|
|
return fmt.Errorf("invalid request id")
|
|
}
|
|
if req.ActionID == "" || len(req.ActionID) > maxRequestIDLength {
|
|
return fmt.Errorf("invalid action id")
|
|
}
|
|
if req.Operation != HostStorageCleanupOperationPackageCache {
|
|
return fmt.Errorf("unsupported host storage cleanup operation %q", req.Operation)
|
|
}
|
|
if req.OperationVersion != HostAPTOperationVersion {
|
|
return fmt.Errorf("unsupported host storage cleanup operation version %d", req.OperationVersion)
|
|
}
|
|
expectedDigest, err := hostStorageCleanupRequestDigest(*req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if req.RequestDigest != expectedDigest {
|
|
return fmt.Errorf("host storage cleanup request digest mismatch")
|
|
}
|
|
if !hostStorageCleanupFingerprintPattern.MatchString(req.ExpectedFingerprint) {
|
|
return fmt.Errorf("expected cleanup fingerprint is required and must be sha256")
|
|
}
|
|
if req.Timeout < 0 || req.Timeout > 900 {
|
|
return fmt.Errorf("host storage cleanup timeout must be between 0 and 900 seconds")
|
|
}
|
|
if req.Timeout == 0 {
|
|
req.Timeout = 300
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateHostStorageCleanupResultPayload(result *HostStorageCleanupResultPayload) error {
|
|
if result == nil {
|
|
return fmt.Errorf("host storage cleanup result is required")
|
|
}
|
|
result.RequestID = strings.TrimSpace(result.RequestID)
|
|
result.Verification = strings.TrimSpace(result.Verification)
|
|
result.ReasonCode = strings.TrimSpace(result.ReasonCode)
|
|
if result.RequestID == "" || len(result.RequestID) > maxRequestIDLength {
|
|
return fmt.Errorf("invalid request id")
|
|
}
|
|
if result.Before.ReclaimableBytes < 0 || result.After.ReclaimableBytes < 0 || result.ReclaimedBytes < 0 {
|
|
return fmt.Errorf("storage cleanup byte counts cannot be negative")
|
|
}
|
|
if result.Before.ReclaimableBytes > HostStorageCleanupMaxReportedBytes || result.After.ReclaimableBytes > HostStorageCleanupMaxReportedBytes || result.ReclaimedBytes > HostStorageCleanupMaxReportedBytes {
|
|
return fmt.Errorf("storage cleanup byte counts exceed bounded limit")
|
|
}
|
|
for _, fingerprint := range []string{result.Before.Fingerprint, result.After.Fingerprint} {
|
|
if fingerprint != "" && !hostStorageCleanupFingerprintPattern.MatchString(fingerprint) {
|
|
return fmt.Errorf("invalid storage cleanup fingerprint")
|
|
}
|
|
}
|
|
switch result.Verification {
|
|
case HostStorageCleanupVerificationVerified:
|
|
if !result.Success || !result.After.Supported || result.After.Provider != "apt-package-cache" || result.After.Error != "" || result.After.Fingerprint == "" {
|
|
return fmt.Errorf("verified storage cleanup lacks a valid postcondition")
|
|
}
|
|
if result.Before.ReclaimableBytes == 0 {
|
|
if result.After.ReclaimableBytes != 0 || result.ReclaimedBytes != 0 {
|
|
return fmt.Errorf("already-satisfied storage cleanup has inconsistent byte counts")
|
|
}
|
|
} else if result.ReclaimedBytes <= 0 || result.After.ReclaimableBytes >= result.Before.ReclaimableBytes || result.ReclaimedBytes != result.Before.ReclaimableBytes-result.After.ReclaimableBytes {
|
|
return fmt.Errorf("verified storage cleanup did not reclaim reported bytes")
|
|
}
|
|
case HostStorageCleanupVerificationFailed, HostStorageCleanupVerificationInconclusive:
|
|
default:
|
|
return fmt.Errorf("unsupported host storage cleanup verification %q", result.Verification)
|
|
}
|
|
if len(result.Error) > 1024 {
|
|
return fmt.Errorf("host storage cleanup error exceeds bounded limit")
|
|
}
|
|
if result.ReasonCode != "" && !IsActionRefusalReasonCode(result.ReasonCode) {
|
|
return fmt.Errorf("invalid host storage cleanup refusal reason code")
|
|
}
|
|
if result.ReasonCode != "" && (result.MutationStarted || result.Success) {
|
|
return fmt.Errorf("host storage cleanup refusal reason conflicts with execution state")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func isAllowedWebSocketOrigin(r *http.Request) bool {
|
|
origin := strings.TrimSpace(r.Header.Get("Origin"))
|
|
if origin == "" {
|
|
return false
|
|
}
|
|
|
|
return securityutil.SameHostWebSocketOrigin(origin, r.Host)
|
|
}
|
|
|
|
func normalizeWebSocketRemoteIP(remoteAddr string) string {
|
|
remoteAddr = strings.TrimSpace(remoteAddr)
|
|
if remoteAddr == "" {
|
|
return ""
|
|
}
|
|
|
|
host, _, err := net.SplitHostPort(remoteAddr)
|
|
if err == nil {
|
|
return strings.Trim(host, "[]")
|
|
}
|
|
|
|
return strings.Trim(remoteAddr, "[]")
|
|
}
|
|
|
|
func (s *Server) acquireWebSocketIPSlot(remoteIP string) bool {
|
|
if s == nil || s.maxConnsPerIP <= 0 || remoteIP == "" {
|
|
return true
|
|
}
|
|
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
if s.ipConnCounts[remoteIP] >= s.maxConnsPerIP {
|
|
return false
|
|
}
|
|
|
|
s.ipConnCounts[remoteIP]++
|
|
return true
|
|
}
|
|
|
|
func (s *Server) releaseWebSocketIPSlot(remoteIP string) {
|
|
if s == nil || s.maxConnsPerIP <= 0 || remoteIP == "" {
|
|
return
|
|
}
|
|
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
count := s.ipConnCounts[remoteIP]
|
|
if count <= 1 {
|
|
delete(s.ipConnCounts, remoteIP)
|
|
return
|
|
}
|
|
|
|
s.ipConnCounts[remoteIP] = count - 1
|
|
}
|
|
|
|
// HandleWebSocket handles incoming WebSocket connections from agents
|
|
func (s *Server) HandleWebSocket(w http.ResponseWriter, r *http.Request) {
|
|
remoteAddr := r.RemoteAddr
|
|
remoteIP := normalizeWebSocketRemoteIP(remoteAddr)
|
|
|
|
if s.isShuttingDown() {
|
|
http.Error(w, "agent execution server is shutting down", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
if !s.acquireWebSocketIPSlot(remoteIP) {
|
|
log.Warn().
|
|
Str("remote_ip", remoteIP).
|
|
Int("max_connections_per_ip", s.maxConnsPerIP).
|
|
Msg("Rejected agent websocket upgrade due to per-IP connection cap")
|
|
http.Error(w, "Too many agent websocket connections from this IP", http.StatusTooManyRequests)
|
|
return
|
|
}
|
|
defer s.releaseWebSocketIPSlot(remoteIP)
|
|
|
|
// CRITICAL: Clear http.Server deadlines BEFORE WebSocket upgrade.
|
|
// The http.Server.ReadTimeout sets a deadline on the underlying connection when
|
|
// the request starts. We must clear it before the upgrade or the connection will
|
|
// be closed when that deadline fires (typically ~15 seconds after connection).
|
|
// Use http.ResponseController (Go 1.20+) to clear the deadline.
|
|
rc := http.NewResponseController(w)
|
|
if err := rc.SetReadDeadline(time.Time{}); err != nil {
|
|
log.Debug().
|
|
Err(err).
|
|
Str("remote_addr", remoteAddr).
|
|
Msg("Failed to clear read deadline via ResponseController")
|
|
}
|
|
if err := rc.SetWriteDeadline(time.Time{}); err != nil {
|
|
log.Debug().
|
|
Err(err).
|
|
Str("remote_addr", remoteAddr).
|
|
Msg("Failed to clear write deadline via ResponseController")
|
|
}
|
|
|
|
conn, err := upgrader.Upgrade(w, r, nil)
|
|
if err != nil {
|
|
log.Error().Err(err).Str("remote_addr", remoteAddr).Msg("Failed to upgrade WebSocket connection")
|
|
return
|
|
}
|
|
conn.SetReadLimit(maxWebSocketMessageBytes)
|
|
closeConn := func(context string) {
|
|
if closeErr := conn.Close(); closeErr != nil {
|
|
log.Debug().Err(closeErr).Msg(context)
|
|
}
|
|
}
|
|
|
|
if s.isShuttingDown() {
|
|
conn.Close()
|
|
return
|
|
}
|
|
|
|
// Also clear on the WebSocket's underlying connection as a safety net
|
|
if netConn := conn.NetConn(); netConn != nil {
|
|
if err := netConn.SetReadDeadline(time.Time{}); err != nil {
|
|
log.Debug().Err(err).Msg("Failed to clear net.Conn read deadline")
|
|
}
|
|
if err := netConn.SetWriteDeadline(time.Time{}); err != nil {
|
|
log.Debug().Err(err).Msg("Failed to clear net.Conn write deadline")
|
|
}
|
|
}
|
|
|
|
// Read first message (must be agent_register)
|
|
if err := conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil {
|
|
log.Warn().Err(err).Msg("Failed to set initial registration read deadline")
|
|
}
|
|
_, msgBytes, err := conn.ReadMessage()
|
|
if err != nil {
|
|
log.Error().Err(err).Str("remote_addr", remoteAddr).Msg("Failed to read registration message")
|
|
closeConn("Failed to close connection after registration read error")
|
|
return
|
|
}
|
|
|
|
var msg Message
|
|
if err := json.Unmarshal(msgBytes, &msg); err != nil {
|
|
log.Error().Err(err).Str("remote_addr", remoteAddr).Msg("Failed to parse registration message")
|
|
closeConn("Failed to close connection after registration parse error")
|
|
return
|
|
}
|
|
|
|
if msg.Type != MsgTypeAgentRegister {
|
|
log.Error().Str("type", string(msg.Type)).Str("remote_addr", remoteAddr).Msg("First message must be agent_register")
|
|
closeConn("Failed to close connection after invalid first message type")
|
|
return
|
|
}
|
|
|
|
// Parse registration payload
|
|
var reg AgentRegisterPayload
|
|
if err := msg.DecodePayload(®); err != nil {
|
|
log.Error().Err(err).Str("remote_addr", remoteAddr).Msg("Failed to parse registration payload")
|
|
closeConn("Failed to close connection after registration payload parse error")
|
|
return
|
|
}
|
|
|
|
reg.AgentID = strings.TrimSpace(reg.AgentID)
|
|
if reg.AgentID == "" {
|
|
log.Warn().Msg("Agent registration rejected: missing agent_id")
|
|
rejMsg, rejErr := NewMessage(MsgTypeRegistered, "", RegisteredPayload{Success: false, Message: "Invalid agent_id"})
|
|
if rejErr != nil {
|
|
log.Warn().Err(rejErr).Msg("Failed to encode rejection message")
|
|
} else if sendErr := s.sendMessage(conn, rejMsg); sendErr != nil {
|
|
log.Warn().Err(sendErr).Msg("Failed to send rejection to agent with missing agent_id")
|
|
}
|
|
conn.Close()
|
|
return
|
|
}
|
|
if len(reg.AgentID) > maxAgentIDLength {
|
|
log.Warn().
|
|
Int("agent_id_length", len(reg.AgentID)).
|
|
Msg("Agent registration rejected: agent_id exceeds maximum length")
|
|
rejMsg, rejErr := NewMessage(MsgTypeRegistered, "", RegisteredPayload{Success: false, Message: "Invalid agent_id"})
|
|
if rejErr != nil {
|
|
log.Warn().Err(rejErr).Msg("Failed to encode rejection for oversized agent_id")
|
|
} else if sendErr := s.sendMessage(conn, rejMsg); sendErr != nil {
|
|
log.Warn().Err(sendErr).Msg("Failed to send rejection to agent with oversized agent_id")
|
|
}
|
|
conn.Close()
|
|
return
|
|
}
|
|
|
|
// Validate and canonicalize the command-session admission. Reporting and
|
|
// command admission are intentionally separate trust decisions.
|
|
admission, admitted := s.admitToken(reg.Token, reg.AgentID, reg.Hostname)
|
|
admission.OrganizationID = normalizeOrganizationID(admission.OrganizationID)
|
|
admission.TokenID = strings.TrimSpace(admission.TokenID)
|
|
admission.AgentID = strings.TrimSpace(admission.AgentID)
|
|
admission.Hostname = strings.TrimSpace(admission.Hostname)
|
|
admission.RuntimeRole = strings.TrimSpace(admission.RuntimeRole)
|
|
admission.ActionCapability = strings.TrimSpace(admission.ActionCapability)
|
|
if admission.AgentID == "" {
|
|
admission.AgentID = reg.AgentID
|
|
}
|
|
if admission.Hostname == "" {
|
|
admission.Hostname = strings.TrimSpace(reg.Hostname)
|
|
}
|
|
if admission.AgentID != reg.AgentID ||
|
|
!unifiedresources.HostnamesEquivalent(admission.Hostname, reg.Hostname) {
|
|
admitted = false
|
|
}
|
|
registrationRole := strings.TrimSpace(reg.RuntimeRole)
|
|
registrationCapability := strings.TrimSpace(reg.ActionCapability)
|
|
if admission.RuntimeRole == RuntimeRoleActionRunner {
|
|
if registrationRole != RuntimeRoleActionRunner ||
|
|
admission.ActionCapability != ActionCapabilityTypedV1 ||
|
|
registrationCapability != admission.ActionCapability {
|
|
admitted = false
|
|
}
|
|
} else if registrationRole == RuntimeRoleActionRunner || registrationCapability != "" {
|
|
// A legacy collector token cannot opt itself into the runner protocol by
|
|
// asserting registration fields that were not bound into its credential.
|
|
admitted = false
|
|
}
|
|
if !admitted {
|
|
log.Warn().Str("agent_id", reg.AgentID).Msg("Agent registration rejected: invalid token")
|
|
// Actionable message instead of a bare "Invalid token": the agent logs
|
|
// this verbatim, and the dominant causes (token not recognised, or not
|
|
// bound to this agent) are both fixed by re-enrolling, while a token
|
|
// that exists but lacks the scope is named explicitly. Avoids the silent
|
|
// retry loop that previously gave operators nothing to act on.
|
|
rejectedMsg, err := NewMessage(MsgTypeRegistered, "", RegisteredPayload{Success: false, Message: "agent token not authorized for command execution — generate a fresh install command with commands enabled (Settings > Infrastructure > Add Pulse Agent) and re-run it on this host; the existing token cannot be upgraded in place"})
|
|
if err != nil {
|
|
log.Warn().Err(err).Str("agent_id", reg.AgentID).Msg("Failed to encode rejection message")
|
|
conn.Close()
|
|
return
|
|
}
|
|
if err := s.sendMessage(conn, rejectedMsg); err != nil {
|
|
log.Warn().Err(err).Str("agent_id", reg.AgentID).Msg("Failed to send rejection to agent")
|
|
}
|
|
closeConn("Failed to close connection after registration rejection")
|
|
return
|
|
}
|
|
|
|
// Create agent connection
|
|
ac := &agentConn{
|
|
conn: conn,
|
|
agent: ConnectedAgent{
|
|
OrganizationID: admission.OrganizationID,
|
|
TokenID: admission.TokenID,
|
|
AgentID: admission.AgentID,
|
|
Hostname: admission.Hostname,
|
|
Version: reg.Version,
|
|
Platform: reg.Platform,
|
|
Tags: reg.Tags,
|
|
RuntimeRole: admission.RuntimeRole,
|
|
ActionCapability: admission.ActionCapability,
|
|
ConnectedAt: time.Now(),
|
|
OperationReceiptVersion: reg.OperationReceiptVersion,
|
|
ActionPreflightVersion: reg.ActionPreflightVersion,
|
|
DockerObservationVersion: reg.DockerObservationVersion,
|
|
},
|
|
admission: admission,
|
|
sessionKey: agentSessionKey(admission.OrganizationID, admission.AgentID),
|
|
authorityKey: uuid.NewString(),
|
|
approvalGrantKey: DeriveApprovalGrantKey(reg.Token),
|
|
done: make(chan struct{}),
|
|
}
|
|
|
|
// Clear deadline for normal operation - both on the WebSocket and underlying connection
|
|
// This MUST happen BEFORE registering the agent in the map to avoid race conditions
|
|
// where other goroutines could call ExecuteCommand while we're still configuring the connection.
|
|
if err := conn.SetReadDeadline(time.Time{}); err != nil {
|
|
log.Warn().Err(err).Str("agent_id", reg.AgentID).Msg("Failed to clear read deadline after registration")
|
|
}
|
|
if err := conn.SetWriteDeadline(time.Time{}); err != nil {
|
|
log.Warn().Err(err).Str("agent_id", reg.AgentID).Msg("Failed to clear write deadline after registration")
|
|
}
|
|
if netConn := conn.NetConn(); netConn != nil {
|
|
if err := netConn.SetReadDeadline(time.Time{}); err != nil {
|
|
log.Warn().Err(err).Str("agent_id", reg.AgentID).Msg("Failed to clear net.Conn read deadline after registration")
|
|
}
|
|
if err := netConn.SetWriteDeadline(time.Time{}); err != nil {
|
|
log.Warn().Err(err).Str("agent_id", reg.AgentID).Msg("Failed to clear net.Conn write deadline after registration")
|
|
}
|
|
}
|
|
|
|
// Set up ping/pong handlers to keep connection alive
|
|
conn.SetPongHandler(func(appData string) error {
|
|
// Reset read deadline on pong received
|
|
if err := conn.SetReadDeadline(time.Time{}); err != nil {
|
|
return fmt.Errorf("set read deadline on pong: %w", err)
|
|
}
|
|
return nil
|
|
})
|
|
|
|
// Register agent - after this point, other goroutines can access the connection
|
|
s.mu.Lock()
|
|
now := time.Now()
|
|
if s.now != nil {
|
|
now = s.now()
|
|
}
|
|
for key, expiry := range s.actionRunnerAdmissionTombstones {
|
|
if !expiry.After(now) {
|
|
delete(s.actionRunnerAdmissionTombstones, key)
|
|
}
|
|
}
|
|
if expiry, cancelled := s.actionRunnerAdmissionTombstones[actionRunnerAdmissionTombstoneKey(admission)]; cancelled && expiry.After(now) {
|
|
s.mu.Unlock()
|
|
log.Warn().Str("agent_id", reg.AgentID).Msg("Action runner registration rejected: prepared credential was cancelled")
|
|
rejectedMsg, err := NewMessage(MsgTypeRegistered, "", RegisteredPayload{Success: false, Message: "action runner credential preparation was cancelled"})
|
|
if err == nil {
|
|
_ = s.sendMessage(conn, rejectedMsg)
|
|
}
|
|
closeConn("Failed to close cancelled action runner registration")
|
|
return
|
|
}
|
|
for key, existing := range s.agents {
|
|
if key != ac.sessionKey &&
|
|
normalizeOrganizationID(existing.admission.OrganizationID) == admission.OrganizationID &&
|
|
unifiedresources.HostnamesEquivalent(existing.agent.Hostname, ac.agent.Hostname) {
|
|
s.mu.Unlock()
|
|
log.Warn().
|
|
Str("organization_id", admission.OrganizationID).
|
|
Str("connected_agent_id", existing.agent.AgentID).
|
|
Str("requested_agent_id", admission.AgentID).
|
|
Str("hostname", admission.Hostname).
|
|
Msg("Agent registration rejected: hostname is already owned by another command identity")
|
|
rejectedMsg, err := NewMessage(MsgTypeRegistered, "", RegisteredPayload{Success: false, Message: "agent hostname is already connected under another identity"})
|
|
if err == nil {
|
|
_ = s.sendMessage(conn, rejectedMsg)
|
|
}
|
|
closeConn("Failed to close duplicate agent hostname connection")
|
|
return
|
|
}
|
|
}
|
|
var replaced *agentConn
|
|
if admission.ActivationPending && admission.RuntimeRole == RuntimeRoleActionRunner {
|
|
// A prepared transport is staged separately. Reconnect/flood traffic can
|
|
// replace only the one bounded pending slot and cannot evict or interrupt
|
|
// the active dispatch session before durable activation.
|
|
replaced = s.pendingActionRunners[ac.sessionKey]
|
|
s.pendingActionRunners[ac.sessionKey] = ac
|
|
} else {
|
|
if existing, ok := s.agents[ac.sessionKey]; ok {
|
|
if !unifiedresources.HostnamesEquivalent(existing.agent.Hostname, ac.agent.Hostname) {
|
|
s.mu.Unlock()
|
|
log.Warn().
|
|
Str("organization_id", admission.OrganizationID).
|
|
Str("agent_id", admission.AgentID).
|
|
Str("connected_hostname", existing.agent.Hostname).
|
|
Str("requested_hostname", admission.Hostname).
|
|
Msg("Agent registration rejected: duplicate identity is already connected from another host")
|
|
rejectedMsg, err := NewMessage(MsgTypeRegistered, "", RegisteredPayload{Success: false, Message: "agent identity is already connected from another host"})
|
|
if err == nil {
|
|
_ = s.sendMessage(conn, rejectedMsg)
|
|
}
|
|
closeConn("Failed to close duplicate agent identity connection")
|
|
return
|
|
}
|
|
replaced = existing
|
|
}
|
|
s.agents[ac.sessionKey] = ac
|
|
}
|
|
s.mu.Unlock()
|
|
if replaced != nil && replaced != ac {
|
|
log.Info().
|
|
Str("organization_id", admission.OrganizationID).
|
|
Str("agent_id", admission.AgentID).
|
|
Str("hostname", admission.Hostname).
|
|
Bool("activation_pending", admission.ActivationPending).
|
|
Msg("Replacing existing agent connection")
|
|
replaced.signalDone()
|
|
if replaced.conn != nil {
|
|
if err := replaced.conn.Close(); err != nil {
|
|
log.Debug().Err(err).Str("agent_id", admission.AgentID).Msg("Failed to close existing connection during reconnect")
|
|
}
|
|
}
|
|
}
|
|
|
|
log.Info().
|
|
Str("organization_id", admission.OrganizationID).
|
|
Str("agent_id", admission.AgentID).
|
|
Str("hostname", admission.Hostname).
|
|
Str("version", reg.Version).
|
|
Str("platform", reg.Platform).
|
|
Msg("Agent connected")
|
|
|
|
// Send registration success
|
|
ackMsg, ackErr := NewMessage(MsgTypeRegistered, "", RegisteredPayload{Success: true, Message: "Registered"})
|
|
if ackErr != nil {
|
|
log.Warn().Err(ackErr).Str("agent_id", reg.AgentID).Msg("Failed to encode registration ack")
|
|
conn.Close()
|
|
return
|
|
}
|
|
ac.writeMu.Lock()
|
|
if sendErr := s.sendMessage(conn, ackMsg); sendErr != nil {
|
|
log.Warn().
|
|
Err(sendErr).
|
|
Str("agent_id", reg.AgentID).
|
|
Str("hostname", reg.Hostname).
|
|
Msg("Failed to send registration ack")
|
|
ac.writeMu.Unlock()
|
|
s.mu.Lock()
|
|
if existing, ok := s.pendingActionRunners[ac.sessionKey]; ok && existing == ac {
|
|
delete(s.pendingActionRunners, ac.sessionKey)
|
|
}
|
|
if existing, ok := s.agents[ac.sessionKey]; ok && existing == ac {
|
|
delete(s.agents, ac.sessionKey)
|
|
}
|
|
s.mu.Unlock()
|
|
ac.signalDone()
|
|
_ = conn.Close()
|
|
return
|
|
}
|
|
ac.writeMu.Unlock()
|
|
|
|
// Start server-side ping loop to keep connection alive
|
|
pingDone := make(chan struct{})
|
|
go s.pingLoop(ac, pingDone)
|
|
defer close(pingDone)
|
|
|
|
if notify := s.agentRegisteredNotifier; notify != nil {
|
|
go notify(admission)
|
|
}
|
|
|
|
// Run read loop (blocking) - don't use goroutine, or HTTP handler will close connection
|
|
s.readLoop(ac)
|
|
}
|
|
|
|
func (s *Server) readLoop(ac *agentConn) {
|
|
defer func() {
|
|
agentID := ac.agent.AgentID
|
|
sessionKey := connectionSessionKey(ac)
|
|
s.mu.Lock()
|
|
wasActive := false
|
|
ownsSession := false
|
|
if existing, exists := s.agents[sessionKey]; exists && existing == ac {
|
|
delete(s.agents, sessionKey)
|
|
wasActive = true
|
|
ownsSession = true
|
|
}
|
|
if existing, exists := s.pendingActionRunners[sessionKey]; exists && existing == ac {
|
|
delete(s.pendingActionRunners, sessionKey)
|
|
ownsSession = true
|
|
}
|
|
// Close all deploy progress subscriptions for this agent so
|
|
// processPreflightProgress goroutines unblock and detect disconnect.
|
|
var closeChs []chan DeployProgressPayload
|
|
if ownsSession && wasActive {
|
|
prefix := sessionKey + "\x00"
|
|
for key, ch := range s.deploySubs {
|
|
if strings.HasPrefix(key, prefix) {
|
|
closeChs = append(closeChs, ch)
|
|
delete(s.deploySubs, key)
|
|
}
|
|
}
|
|
}
|
|
s.mu.Unlock()
|
|
for _, ch := range closeChs {
|
|
close(ch)
|
|
}
|
|
if err := ac.conn.Close(); err != nil {
|
|
log.Debug().Err(err).Str("agent_id", agentID).Msg("Failed to close connection during read-loop cleanup")
|
|
}
|
|
log.Info().Str("agent_id", agentID).Msg("Agent disconnected")
|
|
}()
|
|
|
|
log.Debug().Str("agent_id", ac.agent.AgentID).Msg("Starting read loop for agent")
|
|
|
|
for {
|
|
select {
|
|
case <-ac.done:
|
|
log.Debug().Str("agent_id", ac.agent.AgentID).Msg("Read loop exiting: done channel closed")
|
|
return
|
|
default:
|
|
}
|
|
|
|
_, msgBytes, err := ac.conn.ReadMessage()
|
|
if err != nil {
|
|
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {
|
|
log.Error().Err(err).Str("agent_id", ac.agent.AgentID).Msg("Unexpected WebSocket close error")
|
|
} else {
|
|
log.Debug().Err(err).Str("agent_id", ac.agent.AgentID).Msg("Read loop exiting: read error")
|
|
}
|
|
return
|
|
}
|
|
|
|
var msg Message
|
|
if err := json.Unmarshal(msgBytes, &msg); err != nil {
|
|
log.Error().Err(err).Str("agent_id", ac.agent.AgentID).Msg("Failed to parse message")
|
|
continue
|
|
}
|
|
|
|
// A prepared action runner may prove liveness while activation is
|
|
// pending, but it has no authority to satisfy work dispatched to the
|
|
// active predecessor. Recheck exact active-map membership in each result
|
|
// handler as well, under the same lock as delivery, because promotion can
|
|
// race this early rejection boundary.
|
|
if msg.Type != MsgTypeAgentPing {
|
|
s.mu.RLock()
|
|
active := s.activeConnectionLocked(ac)
|
|
s.mu.RUnlock()
|
|
if !active {
|
|
log.Warn().
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Str("message_type", string(msg.Type)).
|
|
Msg("Dropping inbound message from non-active agent session")
|
|
continue
|
|
}
|
|
}
|
|
|
|
switch msg.Type {
|
|
case MsgTypeAgentPing:
|
|
pongMsg, err := NewMessage(MsgTypePong, "", nil)
|
|
if err != nil {
|
|
log.Debug().Err(err).Str("agent_id", ac.agent.AgentID).Msg("Failed to encode pong message")
|
|
continue
|
|
}
|
|
ac.writeMu.Lock()
|
|
if err := s.sendMessage(ac.conn, pongMsg); err != nil {
|
|
log.Debug().Err(err).Str("agent_id", ac.agent.AgentID).Msg("Failed to send pong")
|
|
}
|
|
ac.writeMu.Unlock()
|
|
|
|
case MsgTypeCommandResult:
|
|
var result CommandResultPayload
|
|
if err := msg.DecodePayload(&result); err != nil {
|
|
log.Error().Err(err).Str("agent_id", ac.agent.AgentID).Msg("Failed to parse command result")
|
|
continue
|
|
}
|
|
result.RequestID = strings.TrimSpace(result.RequestID)
|
|
if result.RequestID == "" {
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Msg("Dropping command result with empty request_id")
|
|
continue
|
|
}
|
|
if len(result.RequestID) > maxRequestIDLength {
|
|
log.Warn().
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Int("request_id_length", len(result.RequestID)).
|
|
Msg("Dropping command result with oversized request_id")
|
|
continue
|
|
}
|
|
|
|
s.mu.RLock()
|
|
ch, ok := s.pendingReqs[pendingRequestKey(connectionAuthorityKey(ac), result.RequestID)]
|
|
active := s.activeConnectionLocked(ac)
|
|
if active && ok {
|
|
select {
|
|
case ch <- result:
|
|
log.Debug().
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Str("request_id", result.RequestID).
|
|
Bool("success", result.Success).
|
|
Int("exit_code", result.ExitCode).
|
|
Int64("duration_ms", result.Duration).
|
|
Msg("Received command result from agent")
|
|
default:
|
|
log.Warn().
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Str("request_id", result.RequestID).
|
|
Msg("Result channel full, dropping")
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
if !active || !ok {
|
|
log.Warn().
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Str("request_id", result.RequestID).
|
|
Msg("No pending request for result")
|
|
}
|
|
|
|
case MsgTypeHostUpdateResult:
|
|
result, decodeErr := DecodeHostUpdateResultPayload(msg.Payload)
|
|
if decodeErr != nil {
|
|
log.Warn().Err(decodeErr).Str("agent_id", ac.agent.AgentID).Msg("Dropping invalid host update result")
|
|
continue
|
|
}
|
|
if !s.matchesPendingHostOperation(connectionAuthorityKey(ac), result.RequestID, result.ActionID, HostUpdateOperationInstall) {
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Dropping uncorrelated host update result")
|
|
continue
|
|
}
|
|
s.mu.RLock()
|
|
ch, ok := s.pendingHostUpdates[pendingRequestKey(connectionAuthorityKey(ac), result.RequestID)]
|
|
if s.activeConnectionLocked(ac) && ok {
|
|
select {
|
|
case ch <- result:
|
|
default:
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Host update result channel full, dropping")
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
|
|
case MsgTypeActionPreflightResult:
|
|
result, decodeErr := DecodeActionPreflightResultPayload(msg.Payload)
|
|
if decodeErr != nil {
|
|
log.Warn().Err(decodeErr).Str("agent_id", ac.agent.AgentID).Msg("Dropping invalid action preflight result")
|
|
continue
|
|
}
|
|
s.mu.RLock()
|
|
ch, ok := s.pendingActionPreflights[pendingRequestKey(connectionAuthorityKey(ac), result.RequestID)]
|
|
if s.activeConnectionLocked(ac) && ok {
|
|
select {
|
|
case ch <- result:
|
|
default:
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Action preflight result channel full, dropping")
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
|
|
case MsgTypeDockerContainerObserveResult:
|
|
result, decodeErr := DecodeDockerContainerObservationResultPayload(msg.Payload)
|
|
if decodeErr != nil {
|
|
log.Warn().Err(decodeErr).Str("agent_id", ac.agent.AgentID).Msg("Dropping invalid docker container observation result")
|
|
continue
|
|
}
|
|
s.mu.RLock()
|
|
ch, ok := s.pendingDockerContainerObservations[pendingRequestKey(connectionAuthorityKey(ac), result.RequestID)]
|
|
if s.activeConnectionLocked(ac) && ok {
|
|
select {
|
|
case ch <- result:
|
|
default:
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Docker observation result channel full, dropping")
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
|
|
case MsgTypeHostStorageCleanupResult:
|
|
result, decodeErr := DecodeHostStorageCleanupResultPayload(msg.Payload)
|
|
if decodeErr != nil {
|
|
log.Warn().Err(decodeErr).Str("agent_id", ac.agent.AgentID).Msg("Dropping invalid host storage cleanup result")
|
|
continue
|
|
}
|
|
if !s.matchesPendingHostOperation(connectionAuthorityKey(ac), result.RequestID, result.ActionID, HostStorageCleanupOperationPackageCache) {
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Dropping uncorrelated host storage cleanup result")
|
|
continue
|
|
}
|
|
s.mu.RLock()
|
|
ch, ok := s.pendingHostStorageCleanups[pendingRequestKey(connectionAuthorityKey(ac), result.RequestID)]
|
|
if s.activeConnectionLocked(ac) && ok {
|
|
select {
|
|
case ch <- result:
|
|
default:
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Host storage cleanup result channel full, dropping")
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
|
|
case MsgTypeProxmoxGuestLifecycleResult:
|
|
result, decodeErr := DecodeProxmoxGuestLifecycleResultPayload(msg.Payload)
|
|
if decodeErr != nil {
|
|
log.Warn().Err(decodeErr).Str("agent_id", ac.agent.AgentID).Msg("Dropping invalid Proxmox guest lifecycle result")
|
|
continue
|
|
}
|
|
if !s.matchesPendingProxmoxGuestOperationForSession(connectionAuthorityKey(ac), ac.agent.AgentID, result) {
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Dropping uncorrelated Proxmox guest lifecycle result")
|
|
continue
|
|
}
|
|
s.mu.RLock()
|
|
ch, ok := s.pendingProxmoxGuestLifecycles[pendingRequestKey(connectionAuthorityKey(ac), result.RequestID)]
|
|
if s.activeConnectionLocked(ac) && ok {
|
|
select {
|
|
case ch <- result:
|
|
default:
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Proxmox guest lifecycle result channel full, dropping")
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
|
|
case MsgTypeDockerContainerLifecycleResult:
|
|
result, decodeErr := DecodeDockerContainerLifecycleResultPayload(msg.Payload)
|
|
if decodeErr != nil {
|
|
log.Warn().Err(decodeErr).Str("agent_id", ac.agent.AgentID).Msg("Dropping invalid docker container lifecycle result")
|
|
continue
|
|
}
|
|
if !s.matchesPendingDockerOperationForSession(connectionAuthorityKey(ac), ac.agent.AgentID, result) {
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Dropping uncorrelated docker lifecycle result")
|
|
continue
|
|
}
|
|
s.mu.RLock()
|
|
ch, ok := s.pendingDockerContainerLifecycles[pendingRequestKey(connectionAuthorityKey(ac), result.RequestID)]
|
|
if s.activeConnectionLocked(ac) && ok {
|
|
select {
|
|
case ch <- result:
|
|
default:
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Docker lifecycle result channel full, dropping")
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
|
|
case MsgTypeDockerContainerUpdateResult:
|
|
result, decodeErr := DecodeDockerContainerUpdateResultPayload(msg.Payload)
|
|
if decodeErr != nil {
|
|
log.Warn().Err(decodeErr).Str("agent_id", ac.agent.AgentID).Msg("Dropping invalid docker container update result")
|
|
continue
|
|
}
|
|
if !s.matchesPendingDockerUpdateOperationForSession(connectionAuthorityKey(ac), ac.agent.AgentID, result) {
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Dropping uncorrelated docker update result")
|
|
continue
|
|
}
|
|
s.mu.RLock()
|
|
ch, ok := s.pendingDockerContainerUpdates[pendingRequestKey(connectionAuthorityKey(ac), result.RequestID)]
|
|
if s.activeConnectionLocked(ac) && ok {
|
|
select {
|
|
case ch <- result:
|
|
default:
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Str("request_id", result.RequestID).Msg("Docker update result channel full, dropping")
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
|
|
case MsgTypeOperationQueryResult:
|
|
result, decodeErr := operationreceipt.DecodeQueryResult(msg.Payload)
|
|
if decodeErr != nil {
|
|
log.Warn().Err(decodeErr).Str("agent_id", ac.agent.AgentID).Msg("Dropping invalid operation query result")
|
|
continue
|
|
}
|
|
key := pendingRequestKey(connectionAuthorityKey(ac), strings.TrimSpace(msg.ID))
|
|
s.mu.RLock()
|
|
pending, ok := s.pendingOperationQueries[key]
|
|
s.mu.RUnlock()
|
|
if !ok {
|
|
continue
|
|
}
|
|
if result.Record != nil && result.Record.Identity != pending.identity {
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Msg("Dropping mismatched operation query result")
|
|
continue
|
|
}
|
|
if err := ValidateOperationQueryResultForIdentity(result, pending.identity, s.currentTime()); err != nil {
|
|
log.Warn().Err(err).Str("agent_id", ac.agent.AgentID).Msg("Dropping invalid correlated operation query result")
|
|
continue
|
|
}
|
|
s.mu.RLock()
|
|
current, stillPending := s.pendingOperationQueries[key]
|
|
if s.activeConnectionLocked(ac) && stillPending && current.ch == pending.ch && current.identity == pending.identity {
|
|
select {
|
|
case pending.ch <- result:
|
|
default:
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
case MsgTypeDeployProgress:
|
|
var progress DeployProgressPayload
|
|
if err := msg.DecodePayload(&progress); err != nil {
|
|
log.Error().Err(err).Str("agent_id", ac.agent.AgentID).Msg("Failed to parse deploy progress")
|
|
continue
|
|
}
|
|
if progress.JobID == "" {
|
|
log.Warn().Str("agent_id", ac.agent.AgentID).Msg("Dropping deploy progress with empty job_id")
|
|
continue
|
|
}
|
|
|
|
subKey := deploySubKey(connectionSessionKey(ac), progress.JobID)
|
|
|
|
// Hold the read lock across map lookup AND the non-blocking send to
|
|
// prevent UnsubscribeDeployProgress from closing the channel between
|
|
// lookup and send (it needs the write lock to delete + close).
|
|
sent := false
|
|
s.mu.RLock()
|
|
ch, ok := s.deploySubs[subKey]
|
|
active := s.activeConnectionLocked(ac)
|
|
if active && ok {
|
|
select {
|
|
case ch <- progress:
|
|
sent = true
|
|
default:
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
|
|
// Final messages must be delivered — retry with backoff if the
|
|
// initial non-blocking send failed (channel was full).
|
|
if active && ok && !sent && progress.Final {
|
|
deadline := time.After(5 * time.Second)
|
|
ticker := time.NewTicker(50 * time.Millisecond)
|
|
retryLoop:
|
|
for {
|
|
select {
|
|
case <-deadline:
|
|
log.Error().
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Str("job_id", progress.JobID).
|
|
Msg("Deploy final progress send timed out — force-closing subscription")
|
|
// Force-close the subscription so the consumer goroutine
|
|
// unblocks on channel close and can finalize the job.
|
|
s.mu.Lock()
|
|
if closeCh, exists := s.deploySubs[subKey]; s.activeConnectionLocked(ac) && exists {
|
|
delete(s.deploySubs, subKey)
|
|
close(closeCh)
|
|
}
|
|
s.mu.Unlock()
|
|
break retryLoop
|
|
case <-ticker.C:
|
|
s.mu.RLock()
|
|
ch, ok = s.deploySubs[subKey]
|
|
active := s.activeConnectionLocked(ac)
|
|
if !ok || !active {
|
|
s.mu.RUnlock()
|
|
break retryLoop // channel was closed/unsubscribed
|
|
}
|
|
select {
|
|
case ch <- progress:
|
|
sent = true
|
|
s.mu.RUnlock()
|
|
break retryLoop
|
|
default:
|
|
s.mu.RUnlock()
|
|
}
|
|
}
|
|
}
|
|
ticker.Stop()
|
|
} else if active && ok && !sent {
|
|
log.Warn().
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Str("job_id", progress.JobID).
|
|
Msg("Deploy progress channel full, dropping")
|
|
}
|
|
|
|
if ok {
|
|
if sent {
|
|
log.Debug().
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Str("job_id", progress.JobID).
|
|
Str("target_id", progress.TargetID).
|
|
Str("phase", string(progress.Phase)).
|
|
Str("status", string(progress.Status)).
|
|
Bool("final", progress.Final).
|
|
Msg("Received deploy progress from agent")
|
|
}
|
|
} else {
|
|
log.Debug().
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Str("job_id", progress.JobID).
|
|
Msg("No subscriber for deploy progress")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Server) pingLoop(ac *agentConn, done chan struct{}) {
|
|
ticker := time.NewTicker(s.pingInterval)
|
|
defer ticker.Stop()
|
|
|
|
// Track consecutive ping failures to detect dead connections faster
|
|
consecutiveFailures := 0
|
|
const maxConsecutiveFailures = 3
|
|
|
|
for {
|
|
select {
|
|
case <-done:
|
|
return
|
|
case <-ac.done:
|
|
return
|
|
case <-ticker.C:
|
|
ac.writeMu.Lock()
|
|
err := ac.conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(pingWriteWait))
|
|
ac.writeMu.Unlock()
|
|
if err != nil {
|
|
consecutiveFailures++
|
|
log.Warn().
|
|
Err(err).
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Str("hostname", ac.agent.Hostname).
|
|
Int("consecutive_failures", consecutiveFailures).
|
|
Msg("Failed to send ping to agent")
|
|
|
|
if consecutiveFailures >= maxConsecutiveFailures {
|
|
log.Error().
|
|
Err(err).
|
|
Str("agent_id", ac.agent.AgentID).
|
|
Str("hostname", ac.agent.Hostname).
|
|
Int("failures", consecutiveFailures).
|
|
Msg("Agent connection appears dead after multiple ping failures, closing connection")
|
|
|
|
// Close the connection - this will cause readLoop to exit and clean up
|
|
if closeErr := ac.conn.Close(); closeErr != nil {
|
|
log.Debug().Err(closeErr).Str("agent_id", ac.agent.AgentID).Msg("Failed to close dead connection after ping failures")
|
|
}
|
|
return
|
|
}
|
|
} else {
|
|
// Reset failure counter on successful ping
|
|
consecutiveFailures = 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Server) sendMessage(conn *websocket.Conn, msg Message) error {
|
|
msgBytes, err := json.Marshal(msg)
|
|
if err != nil {
|
|
return fmt.Errorf("marshal websocket message: %w", err)
|
|
}
|
|
if err := writeTextMessage(conn, msgBytes); err != nil {
|
|
return fmt.Errorf("write websocket message: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Shutdown gracefully stops the server by closing all active agent connections.
|
|
// The method is idempotent.
|
|
func (s *Server) Shutdown() {
|
|
s.shutdownOnce.Do(func() {
|
|
close(s.shutdown)
|
|
|
|
s.mu.Lock()
|
|
agents := make([]*agentConn, 0, len(s.agents)+len(s.pendingActionRunners))
|
|
for _, ac := range s.agents {
|
|
agents = append(agents, ac)
|
|
}
|
|
for _, ac := range s.pendingActionRunners {
|
|
agents = append(agents, ac)
|
|
}
|
|
s.agents = make(map[string]*agentConn)
|
|
s.pendingActionRunners = make(map[string]*agentConn)
|
|
s.mu.Unlock()
|
|
|
|
for _, ac := range agents {
|
|
ac.signalDone()
|
|
_ = ac.conn.Close()
|
|
}
|
|
})
|
|
}
|
|
|
|
// ExecuteCommand sends a command to an agent and waits for the result
|
|
func (s *Server) ExecuteCommand(ctx context.Context, agentID string, cmd ExecuteCommandPayload) (*CommandResultPayload, error) {
|
|
agentID = strings.TrimSpace(agentID)
|
|
if agentID == "" {
|
|
return nil, fmt.Errorf("agent id is required")
|
|
}
|
|
cmd.RequestID = strings.TrimSpace(cmd.RequestID)
|
|
if cmd.RequestID == "" {
|
|
cmd.RequestID = uuid.New().String()
|
|
}
|
|
if len(cmd.RequestID) > maxRequestIDLength {
|
|
return nil, fmt.Errorf("request id exceeds %d characters", maxRequestIDLength)
|
|
}
|
|
if err := validateExecuteCommandPayload(&cmd); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Never dispatch under a context that has already expired: the send
|
|
// would succeed, this call would return the context error a moment
|
|
// later, and the agent would be left executing a command nobody is
|
|
// waiting for. A caller polling on a dead deadline can otherwise
|
|
// re-issue the same command every cycle while every previous copy is
|
|
// still running on the target host (minipc probe-storm incident,
|
|
// 2026-08-20).
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, fmt.Errorf("command %q not dispatched: %w", cmd.RequestID, err)
|
|
}
|
|
|
|
startedAt := time.Now()
|
|
|
|
ac, ok := s.connectionForContext(ctx, agentID)
|
|
if !ok {
|
|
log.Warn().
|
|
Str("agent_id", agentID).
|
|
Str("request_id", cmd.RequestID).
|
|
Msg("Execute command requested for disconnected agent")
|
|
return nil, fmt.Errorf("agent %s not connected", agentID)
|
|
}
|
|
if err := requireLegacyFullTrustConnection(ac, "execute_command"); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.authorizeCommandPayload(cmd); err != nil {
|
|
return nil, err
|
|
}
|
|
requiresApproval := !cmd.Trusted && s.commandPolicy != nil && s.commandPolicy.Evaluate(cmd.Command) == PolicyRequireApproval
|
|
if requiresApproval {
|
|
if len(ac.approvalGrantKey) == 0 {
|
|
return nil, fmt.Errorf("command approval grant signer is unavailable")
|
|
}
|
|
auth := cmd.authorization
|
|
if err := s.commandAuthorizationVerifier(CommandAuthorizationRequest{
|
|
ApprovalID: cmd.ApprovalID,
|
|
OrgID: auth.OrgID,
|
|
ActionID: auth.ActionID,
|
|
AgentID: agentID,
|
|
Command: cmd.Command,
|
|
TargetType: cmd.TargetType,
|
|
TargetID: cmd.TargetID,
|
|
}); err != nil {
|
|
return nil, fmt.Errorf("command approval authorization rejected: %w", err)
|
|
}
|
|
|
|
// Approval grants are an internal transport credential. Never accept a
|
|
// caller-supplied grant, even if it happens to be structurally valid.
|
|
grant, grantErr := s.newCommandApprovalGrant(ac.approvalGrantKey, agentID, cmd, time.Now(), DefaultApprovalGrantTTL)
|
|
if grantErr != nil {
|
|
return nil, fmt.Errorf("failed to issue approval grant: %w", grantErr)
|
|
}
|
|
cmd.ApprovalGrant = grant
|
|
}
|
|
|
|
execLog := log.With().
|
|
Str("agent_id", agentID).
|
|
Str("request_id", cmd.RequestID).
|
|
Str("target_type", cmd.TargetType).
|
|
Str("target_id", cmd.TargetID).
|
|
Logger()
|
|
|
|
// Create response channel
|
|
respCh := make(chan CommandResultPayload, 1)
|
|
reqKey := pendingRequestKey(connectionAuthorityKey(ac), cmd.RequestID)
|
|
s.mu.Lock()
|
|
if _, exists := s.pendingReqs[reqKey]; exists {
|
|
s.mu.Unlock()
|
|
return nil, fmt.Errorf("command request %q is already pending", cmd.RequestID)
|
|
}
|
|
s.pendingReqs[reqKey] = respCh
|
|
s.mu.Unlock()
|
|
|
|
defer func() {
|
|
s.mu.Lock()
|
|
delete(s.pendingReqs, reqKey)
|
|
s.mu.Unlock()
|
|
}()
|
|
|
|
// Send command
|
|
execMsg, execErr := NewMessage(MsgTypeExecuteCmd, cmd.RequestID, cmd)
|
|
if execErr != nil {
|
|
return nil, fmt.Errorf("failed to encode command: %w", execErr)
|
|
}
|
|
|
|
ac.writeMu.Lock()
|
|
err := s.sendMessage(ac.conn, execMsg)
|
|
ac.writeMu.Unlock()
|
|
|
|
if err != nil {
|
|
execLog.Error().
|
|
Err(err).
|
|
Dur("duration", time.Since(startedAt)).
|
|
Msg("Failed to send command to agent")
|
|
return nil, fmt.Errorf("failed to send command: %w", err)
|
|
}
|
|
|
|
// Wait for result
|
|
timeout := time.Duration(cmd.Timeout) * time.Second
|
|
if timeout <= 0 {
|
|
timeout = 60 * time.Second
|
|
}
|
|
timer := time.NewTimer(timeout)
|
|
defer func() {
|
|
if !timer.Stop() {
|
|
select {
|
|
case <-timer.C:
|
|
default:
|
|
}
|
|
}
|
|
}()
|
|
|
|
select {
|
|
case result := <-respCh:
|
|
execLog.Info().
|
|
Bool("success", result.Success).
|
|
Int("exit_code", result.ExitCode).
|
|
Int64("agent_duration_ms", result.Duration).
|
|
Dur("duration", time.Since(startedAt)).
|
|
Msg("Agent command completed")
|
|
return &result, nil
|
|
case <-timer.C:
|
|
s.cancelAgentRequest(ac, cmd.RequestID, "execute_command")
|
|
execLog.Warn().
|
|
Dur("timeout", timeout).
|
|
Dur("duration", time.Since(startedAt)).
|
|
Msg("Agent command timed out")
|
|
return nil, fmt.Errorf("command timed out after %v", timeout)
|
|
case <-ctx.Done():
|
|
s.cancelAgentRequest(ac, cmd.RequestID, "execute_command")
|
|
execLog.Warn().
|
|
Err(ctx.Err()).
|
|
Dur("duration", time.Since(startedAt)).
|
|
Msg("Agent command canceled")
|
|
return nil, ctx.Err()
|
|
case <-ac.done:
|
|
return nil, fmt.Errorf("agent %s disconnected before command result", agentID)
|
|
case <-s.shutdown:
|
|
return nil, errServerShuttingDown
|
|
}
|
|
}
|
|
|
|
// cancelAgentRequest tells an agent to abort a request the server has stopped
|
|
// waiting for. Typed runners bind this message to the in-flight request
|
|
// context; already-started mutations remain indeterminate and are reconciled
|
|
// through their durable receipts. Best effort: agents that predate the
|
|
// cancel_command message ignore it and fall back to their own timeout.
|
|
func (s *Server) cancelAgentRequest(ac *agentConn, requestID, operation string) {
|
|
msg, err := NewMessage(MsgTypeCancelCmd, requestID, CancelCommandPayload{RequestID: requestID})
|
|
if err != nil {
|
|
log.Debug().Err(err).Str("request_id", requestID).Str("operation", operation).Msg("Failed to encode request cancellation")
|
|
return
|
|
}
|
|
ac.writeMu.Lock()
|
|
err = s.sendMessage(ac.conn, msg)
|
|
ac.writeMu.Unlock()
|
|
if err != nil {
|
|
log.Debug().Err(err).Str("request_id", requestID).Str("operation", operation).Msg("Failed to send request cancellation to agent")
|
|
}
|
|
}
|
|
|
|
// hostOperationPayload exposes the durable operation identity shared by the
|
|
// typed host APT operation request payloads.
|
|
type hostOperationPayload interface {
|
|
hostOperationIdentity() (requestID, actionID, operation string, timeoutSeconds int)
|
|
}
|
|
|
|
func (p HostUpdatePayload) hostOperationIdentity() (string, string, string, int) {
|
|
return p.RequestID, p.ActionID, p.Operation, p.Timeout
|
|
}
|
|
|
|
func (p HostStorageCleanupPayload) hostOperationIdentity() (string, string, string, int) {
|
|
return p.RequestID, p.ActionID, p.Operation, p.Timeout
|
|
}
|
|
|
|
// hostOperationDispatch names the per-operation pieces of the shared typed
|
|
// host-operation dispatch cycle: claim → send → await validated receipt.
|
|
type hostOperationDispatch[Req hostOperationPayload, Res any] struct {
|
|
msgType MessageType
|
|
label string
|
|
pending map[string]chan Res
|
|
validateResult func(Req, Res, time.Time) error
|
|
}
|
|
|
|
// prepareHostOperationRequest runs the shared request prologue of the typed
|
|
// host-operation dispatchers: normalize the agent id, default the request id,
|
|
// then bind and validate the payload. It returns the normalized agent id.
|
|
func prepareHostOperationRequest(s *Server, agentID string, requestID *string, bind func() error, validate func() error) (string, error) {
|
|
if s == nil {
|
|
return "", fmt.Errorf("agent execution server is unavailable")
|
|
}
|
|
agentID = strings.TrimSpace(agentID)
|
|
if agentID == "" {
|
|
return "", fmt.Errorf("agent id is required")
|
|
}
|
|
if strings.TrimSpace(*requestID) == "" {
|
|
*requestID = uuid.New().String()
|
|
}
|
|
if err := bind(); err != nil {
|
|
return "", err
|
|
}
|
|
if err := validate(); err != nil {
|
|
return "", err
|
|
}
|
|
return agentID, nil
|
|
}
|
|
|
|
func dispatchHostOperation[Req hostOperationPayload, Res any](ctx context.Context, s *Server, agentID string, req Req, op hostOperationDispatch[Req, Res]) (*Res, error) {
|
|
requestID, actionID, operation, timeoutSeconds := req.hostOperationIdentity()
|
|
if ctx == nil {
|
|
return nil, fmt.Errorf("%s request %q not dispatched: context is required", op.label, requestID)
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, fmt.Errorf("%s request %q not dispatched: %w", op.label, requestID, err)
|
|
}
|
|
|
|
ac, ok := s.connectionForContext(ctx, agentID)
|
|
if !ok {
|
|
return nil, fmt.Errorf("agent %s not connected", agentID)
|
|
}
|
|
if ac.agent.OperationReceiptVersion != operationreceipt.ProtocolVersion {
|
|
return nil, fmt.Errorf("agent does not support durable operation receipts")
|
|
}
|
|
|
|
respCh := make(chan Res, 1)
|
|
authorityKey := connectionAuthorityKey(ac)
|
|
reqKey := pendingRequestKey(authorityKey, requestID)
|
|
hostOperationKey, err := s.claimPendingHostOperation(authorityKey, requestID, actionID, operation)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer s.releasePendingHostOperation(hostOperationKey)
|
|
s.mu.Lock()
|
|
if _, exists := op.pending[reqKey]; exists {
|
|
s.mu.Unlock()
|
|
return nil, fmt.Errorf("%s request %q is already pending", op.label, requestID)
|
|
}
|
|
op.pending[reqKey] = respCh
|
|
s.mu.Unlock()
|
|
defer func() {
|
|
s.mu.Lock()
|
|
delete(op.pending, reqKey)
|
|
s.mu.Unlock()
|
|
}()
|
|
|
|
msg, err := NewMessage(op.msgType, requestID, req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to encode %s request: %w", op.label, err)
|
|
}
|
|
ac.writeMu.Lock()
|
|
err = s.sendMessage(ac.conn, msg)
|
|
ac.writeMu.Unlock()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to send %s request: %w", op.label, err)
|
|
}
|
|
|
|
timer := time.NewTimer(time.Duration(timeoutSeconds) * time.Second)
|
|
defer timer.Stop()
|
|
select {
|
|
case result := <-respCh:
|
|
if err := op.validateResult(req, result, s.currentTime()); err != nil {
|
|
return nil, fmt.Errorf("%s result validation failed: %w", op.label, err)
|
|
}
|
|
return &result, nil
|
|
case <-timer.C:
|
|
s.cancelAgentRequest(ac, requestID, operation)
|
|
return nil, fmt.Errorf("%s timed out after %s", op.label, time.Duration(timeoutSeconds)*time.Second)
|
|
case <-ctx.Done():
|
|
s.cancelAgentRequest(ac, requestID, operation)
|
|
return nil, ctx.Err()
|
|
case <-ac.done:
|
|
return nil, fmt.Errorf("agent %s disconnected before %s receipt", agentID, op.label)
|
|
case <-s.shutdown:
|
|
return nil, errServerShuttingDown
|
|
}
|
|
}
|
|
|
|
// ExecuteHostUpdate dispatches the closed typed host-package operation. Unlike
|
|
// ExecuteCommand, no command text crosses this boundary; the agent owns the
|
|
// package-manager catalog, preflight, mutation, and read-after-write proof.
|
|
func (s *Server) ExecuteHostUpdate(ctx context.Context, agentID string, req HostUpdatePayload) (*HostUpdateResultPayload, error) {
|
|
agentID, err := prepareHostOperationRequest(s, agentID, &req.RequestID,
|
|
func() error { return BindHostUpdatePayload(&req) },
|
|
func() error { return ValidateHostUpdatePayload(&req) })
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return dispatchHostOperation(ctx, s, agentID, req, hostOperationDispatch[HostUpdatePayload, HostUpdateResultPayload]{
|
|
msgType: MsgTypeHostUpdate, label: "host update",
|
|
pending: s.pendingHostUpdates, validateResult: ValidateHostUpdateResultForRequestAt,
|
|
})
|
|
}
|
|
|
|
// PreflightAction asks the current Unified Agent to evaluate the exact typed
|
|
// operation without admitting a durable operation or starting a mutation.
|
|
func (s *Server) PreflightAction(ctx context.Context, agentID string, req ActionPreflightPayload) (*ActionPreflightResultPayload, error) {
|
|
if s == nil {
|
|
return nil, fmt.Errorf("agent execution server is unavailable")
|
|
}
|
|
agentID = strings.TrimSpace(agentID)
|
|
if agentID == "" {
|
|
return nil, fmt.Errorf("agent id is required")
|
|
}
|
|
if strings.TrimSpace(req.RequestID) == "" {
|
|
req.RequestID = uuid.NewString()
|
|
}
|
|
if err := ValidateActionPreflightPayload(&req); err != nil {
|
|
return nil, err
|
|
}
|
|
ac, ok := s.connectionForContext(ctx, agentID)
|
|
if !ok {
|
|
return nil, fmt.Errorf("agent %s not connected", agentID)
|
|
}
|
|
if ac.agent.ActionPreflightVersion != ActionPreflightProtocolVersion {
|
|
return nil, fmt.Errorf("agent does not support action preflight protocol")
|
|
}
|
|
ch := make(chan ActionPreflightResultPayload, 1)
|
|
key := pendingRequestKey(connectionAuthorityKey(ac), req.RequestID)
|
|
s.mu.Lock()
|
|
if _, exists := s.pendingActionPreflights[key]; exists {
|
|
s.mu.Unlock()
|
|
return nil, fmt.Errorf("action preflight request %q is already pending", req.RequestID)
|
|
}
|
|
s.pendingActionPreflights[key] = ch
|
|
s.mu.Unlock()
|
|
defer func() {
|
|
s.mu.Lock()
|
|
delete(s.pendingActionPreflights, key)
|
|
s.mu.Unlock()
|
|
}()
|
|
msg, err := NewMessage(MsgTypeActionPreflight, req.RequestID, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ac.writeMu.Lock()
|
|
err = s.sendMessage(ac.conn, msg)
|
|
ac.writeMu.Unlock()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to send action preflight request: %w", err)
|
|
}
|
|
timer := time.NewTimer(20 * time.Second)
|
|
defer timer.Stop()
|
|
select {
|
|
case result := <-ch:
|
|
if err := ValidateActionPreflightResultForRequest(req, result, s.currentTime()); err != nil {
|
|
return nil, fmt.Errorf("action preflight result validation failed: %w", err)
|
|
}
|
|
return &result, nil
|
|
case <-timer.C:
|
|
return nil, fmt.Errorf("action preflight timed out")
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case <-ac.done:
|
|
return nil, fmt.Errorf("agent %s disconnected before action preflight result", agentID)
|
|
case <-s.shutdown:
|
|
return nil, errServerShuttingDown
|
|
}
|
|
}
|
|
|
|
// ObserveDockerContainer asks the current Unified Agent for a fresh read-only
|
|
// Docker/Podman daemon observation. It is a separate request from the mutation
|
|
// receipt and carries no dispatch authority.
|
|
func (s *Server) ObserveDockerContainer(ctx context.Context, agentID string, req DockerContainerObservationPayload) (*DockerContainerObservationResultPayload, error) {
|
|
if s == nil {
|
|
return nil, fmt.Errorf("agent execution server is unavailable")
|
|
}
|
|
agentID = strings.TrimSpace(agentID)
|
|
if agentID == "" {
|
|
return nil, fmt.Errorf("agent id is required")
|
|
}
|
|
if strings.TrimSpace(req.RequestID) == "" {
|
|
req.RequestID = uuid.NewString()
|
|
}
|
|
if err := BindDockerContainerObservationPayload(&req); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := ValidateDockerContainerObservationPayload(&req); err != nil {
|
|
return nil, err
|
|
}
|
|
ac, ok := s.connectionForContext(ctx, agentID)
|
|
if !ok {
|
|
return nil, fmt.Errorf("agent %s not connected", agentID)
|
|
}
|
|
if ac.agent.DockerObservationVersion != DockerContainerObservationProtocolVersion {
|
|
return nil, fmt.Errorf("agent does not support docker observation protocol")
|
|
}
|
|
ch := make(chan DockerContainerObservationResultPayload, 1)
|
|
key := pendingRequestKey(connectionAuthorityKey(ac), req.RequestID)
|
|
s.mu.Lock()
|
|
if _, exists := s.pendingDockerContainerObservations[key]; exists {
|
|
s.mu.Unlock()
|
|
return nil, fmt.Errorf("docker observation request %q is already pending", req.RequestID)
|
|
}
|
|
s.pendingDockerContainerObservations[key] = ch
|
|
s.mu.Unlock()
|
|
defer func() {
|
|
s.mu.Lock()
|
|
delete(s.pendingDockerContainerObservations, key)
|
|
s.mu.Unlock()
|
|
}()
|
|
msg, err := NewMessage(MsgTypeDockerContainerObserve, req.RequestID, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ac.writeMu.Lock()
|
|
err = s.sendMessage(ac.conn, msg)
|
|
ac.writeMu.Unlock()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to send docker observation request: %w", err)
|
|
}
|
|
timer := time.NewTimer(20 * time.Second)
|
|
defer timer.Stop()
|
|
select {
|
|
case result := <-ch:
|
|
if err := ValidateDockerContainerObservationResultForRequest(req, result, s.currentTime()); err != nil {
|
|
return nil, fmt.Errorf("docker observation result validation failed: %w", err)
|
|
}
|
|
return &result, nil
|
|
case <-timer.C:
|
|
return nil, fmt.Errorf("docker observation timed out")
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case <-ac.done:
|
|
return nil, fmt.Errorf("agent %s disconnected before docker observation result", agentID)
|
|
case <-s.shutdown:
|
|
return nil, errServerShuttingDown
|
|
}
|
|
}
|
|
|
|
// ExecuteHostStorageCleanup dispatches the closed package-cache cleanup
|
|
// operation. No command text, path, package selector, or removal policy crosses
|
|
// the server/agent boundary.
|
|
func (s *Server) ExecuteHostStorageCleanup(ctx context.Context, agentID string, req HostStorageCleanupPayload) (*HostStorageCleanupResultPayload, error) {
|
|
agentID, err := prepareHostOperationRequest(s, agentID, &req.RequestID,
|
|
func() error { return BindHostStorageCleanupPayload(&req) },
|
|
func() error { return ValidateHostStorageCleanupPayload(&req) })
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return dispatchHostOperation(ctx, s, agentID, req, hostOperationDispatch[HostStorageCleanupPayload, HostStorageCleanupResultPayload]{
|
|
msgType: MsgTypeHostStorageCleanup, label: "host storage cleanup",
|
|
pending: s.pendingHostStorageCleanups, validateResult: ValidateHostStorageCleanupResultForRequestAt,
|
|
})
|
|
}
|
|
|
|
// ExecuteDockerContainerLifecycle dispatches one closed typed container
|
|
// operation. The Unified Agent owns the fixed runtime command catalog and
|
|
// performs preflight plus read-after-write inside this single dispatch.
|
|
func (s *Server) ExecuteDockerContainerLifecycle(ctx context.Context, agentID string, req DockerContainerLifecyclePayload) (*DockerContainerLifecycleResultPayload, error) {
|
|
if s == nil {
|
|
return nil, fmt.Errorf("agent execution server is unavailable")
|
|
}
|
|
agentID = strings.TrimSpace(agentID)
|
|
if agentID == "" {
|
|
return nil, fmt.Errorf("agent id is required")
|
|
}
|
|
if strings.TrimSpace(req.RequestID) == "" {
|
|
req.RequestID = uuid.NewString()
|
|
}
|
|
if err := BindDockerContainerLifecyclePayload(&req); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := ValidateDockerContainerLifecyclePayload(&req); err != nil {
|
|
return nil, err
|
|
}
|
|
identity := DockerContainerLifecycleOperationIdentity(agentID, req)
|
|
return dispatchTypedDockerContainerOperation(ctx, s, agentID, req.RequestID, req.Timeout, identity, req.ContainerID,
|
|
MsgTypeDockerContainerLifecycle, req, s.pendingDockerContainerLifecycles, "docker container lifecycle",
|
|
func(result DockerContainerLifecycleResultPayload) error {
|
|
return ValidateDockerContainerLifecycleResultForRequest(req, result)
|
|
})
|
|
}
|
|
|
|
// ExecuteProxmoxGuestLifecycle dispatches one closed Proxmox guest action to
|
|
// an action-runner session. The wire contract contains only guest kind, fixed
|
|
// lifecycle verb, numeric VMID, and request-bound before state.
|
|
func (s *Server) ExecuteProxmoxGuestLifecycle(ctx context.Context, agentID string, req ProxmoxGuestLifecyclePayload) (*ProxmoxGuestLifecycleResultPayload, error) {
|
|
if s == nil {
|
|
return nil, fmt.Errorf("agent execution server is unavailable")
|
|
}
|
|
agentID = strings.TrimSpace(agentID)
|
|
if agentID == "" {
|
|
return nil, fmt.Errorf("agent id is required")
|
|
}
|
|
if strings.TrimSpace(req.RequestID) == "" {
|
|
req.RequestID = uuid.NewString()
|
|
}
|
|
if err := BindProxmoxGuestLifecyclePayload(&req); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := ValidateProxmoxGuestLifecyclePayload(&req); err != nil {
|
|
return nil, err
|
|
}
|
|
ac, ok := s.connectionForContext(ctx, agentID)
|
|
if !ok {
|
|
return nil, fmt.Errorf("agent %s not connected", agentID)
|
|
}
|
|
if ac.admission.RuntimeRole != RuntimeRoleActionRunner || ac.admission.ActionCapability != ActionCapabilityTypedV1 {
|
|
return nil, fmt.Errorf("Proxmox guest lifecycle requires a typed action-runner session")
|
|
}
|
|
identity := ProxmoxGuestLifecycleOperationIdentity(agentID, req)
|
|
return dispatchTypedDockerContainerOperation(ctx, s, agentID, req.RequestID, req.Timeout, identity, req.GuestKind+":"+strconv.Itoa(req.VMID),
|
|
MsgTypeProxmoxGuestLifecycle, req, s.pendingProxmoxGuestLifecycles, "Proxmox guest lifecycle",
|
|
func(result ProxmoxGuestLifecycleResultPayload) error {
|
|
return ValidateProxmoxGuestLifecycleResultForRequest(req, result)
|
|
})
|
|
}
|
|
|
|
// dispatchTypedDockerContainerOperation owns the shared skeleton for closed
|
|
// typed container dispatches: durable-receipt capability check, pending
|
|
// operation claim, single-flight request registration, send, and the bounded
|
|
// wait for the validated result.
|
|
func dispatchTypedDockerContainerOperation[Res any](
|
|
ctx context.Context, s *Server, agentID, requestID string, timeoutSeconds int,
|
|
identity operationreceipt.Identity, containerID string,
|
|
msgType MessageType, payload any,
|
|
pending map[string]chan Res, label string,
|
|
validate func(Res) error,
|
|
) (*Res, error) {
|
|
if ctx == nil {
|
|
return nil, fmt.Errorf("%s request %q not dispatched: context is required", label, requestID)
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, fmt.Errorf("%s request %q not dispatched: %w", label, requestID, err)
|
|
}
|
|
ac, ok := s.connectionForContext(ctx, agentID)
|
|
if !ok {
|
|
return nil, fmt.Errorf("agent %s not connected", agentID)
|
|
}
|
|
if ac.agent.OperationReceiptVersion != operationreceipt.ProtocolVersion {
|
|
return nil, fmt.Errorf("agent does not support durable operation receipts")
|
|
}
|
|
|
|
respCh := make(chan Res, 1)
|
|
authorityKey := connectionAuthorityKey(ac)
|
|
reqKey := pendingRequestKey(authorityKey, requestID)
|
|
hostOperationKey, err := s.claimPendingDockerOperationForSession(authorityKey, identity, containerID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer s.releasePendingHostOperation(hostOperationKey)
|
|
s.mu.Lock()
|
|
if _, exists := pending[reqKey]; exists {
|
|
s.mu.Unlock()
|
|
return nil, fmt.Errorf("%s request %q is already pending", label, requestID)
|
|
}
|
|
pending[reqKey] = respCh
|
|
s.mu.Unlock()
|
|
defer func() {
|
|
s.mu.Lock()
|
|
delete(pending, reqKey)
|
|
s.mu.Unlock()
|
|
}()
|
|
|
|
msg, err := NewMessage(msgType, requestID, payload)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to encode %s request: %w", label, err)
|
|
}
|
|
ac.writeMu.Lock()
|
|
err = s.sendMessage(ac.conn, msg)
|
|
ac.writeMu.Unlock()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to send %s request: %w", label, err)
|
|
}
|
|
timer := time.NewTimer(time.Duration(timeoutSeconds) * time.Second)
|
|
defer timer.Stop()
|
|
select {
|
|
case result := <-respCh:
|
|
if err := validate(result); err != nil {
|
|
return nil, fmt.Errorf("%s result validation failed: %w", label, err)
|
|
}
|
|
return &result, nil
|
|
case <-timer.C:
|
|
s.cancelAgentRequest(ac, requestID, identity.OperationKind)
|
|
return nil, fmt.Errorf("%s timed out after %s", label, time.Duration(timeoutSeconds)*time.Second)
|
|
case <-ctx.Done():
|
|
s.cancelAgentRequest(ac, requestID, identity.OperationKind)
|
|
return nil, ctx.Err()
|
|
case <-ac.done:
|
|
return nil, fmt.Errorf("agent %s disconnected before %s receipt", agentID, label)
|
|
case <-s.shutdown:
|
|
return nil, errServerShuttingDown
|
|
}
|
|
}
|
|
|
|
// ExecuteDockerContainerUpdate dispatches one closed typed container image
|
|
// update. The Unified Agent owns pull, backup, recreate, verification, and
|
|
// rollback inside this single dispatch and reports the compensation outcome.
|
|
func (s *Server) ExecuteDockerContainerUpdate(ctx context.Context, agentID string, req DockerContainerUpdatePayload) (*DockerContainerUpdateResultPayload, error) {
|
|
if s == nil {
|
|
return nil, fmt.Errorf("agent execution server is unavailable")
|
|
}
|
|
agentID = strings.TrimSpace(agentID)
|
|
if agentID == "" {
|
|
return nil, fmt.Errorf("agent id is required")
|
|
}
|
|
if strings.TrimSpace(req.RequestID) == "" {
|
|
req.RequestID = uuid.NewString()
|
|
}
|
|
if err := BindDockerContainerUpdatePayload(&req); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := ValidateDockerContainerUpdatePayload(&req); err != nil {
|
|
return nil, err
|
|
}
|
|
identity := DockerContainerUpdateOperationIdentity(agentID, req)
|
|
return dispatchTypedDockerContainerOperation(ctx, s, agentID, req.RequestID, req.Timeout, identity, req.ContainerID,
|
|
MsgTypeDockerContainerUpdate, req, s.pendingDockerContainerUpdates, "docker container update",
|
|
func(result DockerContainerUpdateResultPayload) error {
|
|
return ValidateDockerContainerUpdateResultForRequest(req, result)
|
|
})
|
|
}
|
|
|
|
func (s *Server) currentTime() time.Time {
|
|
if s != nil && s.now != nil {
|
|
return s.now().UTC()
|
|
}
|
|
return time.Now().UTC()
|
|
}
|
|
|
|
func (s *Server) AgentOperationReceiptVersion(agentID string) int {
|
|
return s.AgentOperationReceiptVersionForOrganization(defaultOrganizationID, agentID)
|
|
}
|
|
|
|
// AgentOperationReceiptVersionForOrganization reports the live protocol
|
|
// version only for a currently admitted tenant-scoped session.
|
|
func (s *Server) AgentOperationReceiptVersionForOrganization(organizationID, agentID string) int {
|
|
if s == nil {
|
|
return 0
|
|
}
|
|
connection, ok := s.connectionForOrganization(organizationID, agentID)
|
|
if !ok {
|
|
return 0
|
|
}
|
|
return connection.agent.OperationReceiptVersion
|
|
}
|
|
|
|
// QueryAgentOperation reconciles a committed attempt without mutation or resend.
|
|
func (s *Server) QueryAgentOperation(ctx context.Context, agentID string, identity operationreceipt.Identity) (operationreceipt.QueryResult, error) {
|
|
identity, err := operationreceipt.NormalizeIdentity(identity)
|
|
if err != nil {
|
|
return operationreceipt.QueryResult{}, err
|
|
}
|
|
agentID = strings.TrimSpace(agentID)
|
|
if identity.AgentID != agentID {
|
|
return operationreceipt.QueryResult{}, operationreceipt.ErrBindingConflict
|
|
}
|
|
ac, ok := s.connectionForContext(ctx, agentID)
|
|
if !ok {
|
|
return operationreceipt.QueryResult{}, fmt.Errorf("agent %s not connected", agentID)
|
|
}
|
|
if ac.agent.OperationReceiptVersion != operationreceipt.ProtocolVersion {
|
|
return operationreceipt.QueryResult{}, fmt.Errorf("agent does not support durable operation receipts")
|
|
}
|
|
queryID := identity.AttemptID + ".query." + uuid.NewString()
|
|
key := pendingRequestKey(connectionAuthorityKey(ac), queryID)
|
|
ch := make(chan operationreceipt.QueryResult, 1)
|
|
s.mu.Lock()
|
|
if _, exists := s.pendingOperationQueries[key]; exists {
|
|
s.mu.Unlock()
|
|
return operationreceipt.QueryResult{}, fmt.Errorf("operation query %q is already pending", identity.AttemptID)
|
|
}
|
|
s.pendingOperationQueries[key] = pendingOperationQuery{identity: identity, ch: ch}
|
|
s.mu.Unlock()
|
|
defer func() { s.mu.Lock(); delete(s.pendingOperationQueries, key); s.mu.Unlock() }()
|
|
msg, err := NewMessage(MsgTypeOperationQuery, queryID, operationreceipt.Query{Version: operationreceipt.ProtocolVersion, Identity: identity})
|
|
if err != nil {
|
|
return operationreceipt.QueryResult{}, err
|
|
}
|
|
ac.writeMu.Lock()
|
|
err = s.sendMessage(ac.conn, msg)
|
|
ac.writeMu.Unlock()
|
|
if err != nil {
|
|
return operationreceipt.QueryResult{}, err
|
|
}
|
|
timer := time.NewTimer(operationQueryTimeout)
|
|
defer timer.Stop()
|
|
select {
|
|
case result := <-ch:
|
|
return result, nil
|
|
case <-ctx.Done():
|
|
return operationreceipt.QueryResult{}, ctx.Err()
|
|
case <-timer.C:
|
|
return operationreceipt.QueryResult{}, fmt.Errorf("operation receipt query timed out")
|
|
case <-ac.done:
|
|
return operationreceipt.QueryResult{}, fmt.Errorf("agent %s disconnected during operation query", agentID)
|
|
case <-s.shutdown:
|
|
return operationreceipt.QueryResult{}, errServerShuttingDown
|
|
}
|
|
}
|
|
|
|
// ReadFile reads a file from an agent
|
|
func (s *Server) ReadFile(ctx context.Context, agentID string, req ReadFilePayload) (*CommandResultPayload, error) {
|
|
agentID = strings.TrimSpace(agentID)
|
|
if agentID == "" {
|
|
return nil, fmt.Errorf("agent id is required")
|
|
}
|
|
req.RequestID = strings.TrimSpace(req.RequestID)
|
|
if req.RequestID == "" {
|
|
req.RequestID = uuid.New().String()
|
|
}
|
|
if err := validateReadFilePayload(&req); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Same rule as ExecuteCommand: never dispatch work the caller has
|
|
// already stopped waiting for.
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, fmt.Errorf("read_file %q not dispatched: %w", req.RequestID, err)
|
|
}
|
|
|
|
ac, ok := s.connectionForContext(ctx, agentID)
|
|
if !ok {
|
|
log.Warn().
|
|
Str("agent_id", agentID).
|
|
Str("request_id", req.RequestID).
|
|
Msg("Read file requested for disconnected agent")
|
|
return nil, fmt.Errorf("agent %s not connected", agentID)
|
|
}
|
|
if err := requireLegacyFullTrustConnection(ac, "read_file"); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
readLog := log.With().
|
|
Str("agent_id", agentID).
|
|
Str("request_id", req.RequestID).
|
|
Str("path", req.Path).
|
|
Str("target_type", req.TargetType).
|
|
Str("target_id", req.TargetID).
|
|
Int64("max_bytes", req.MaxBytes).
|
|
Logger()
|
|
|
|
startedAt := time.Now()
|
|
|
|
// Create response channel
|
|
respCh := make(chan CommandResultPayload, 1)
|
|
reqKey := pendingRequestKey(connectionAuthorityKey(ac), req.RequestID)
|
|
s.mu.Lock()
|
|
if _, exists := s.pendingReqs[reqKey]; exists {
|
|
s.mu.Unlock()
|
|
return nil, fmt.Errorf("read_file request %q is already pending", req.RequestID)
|
|
}
|
|
s.pendingReqs[reqKey] = respCh
|
|
s.mu.Unlock()
|
|
|
|
defer func() {
|
|
s.mu.Lock()
|
|
delete(s.pendingReqs, reqKey)
|
|
s.mu.Unlock()
|
|
}()
|
|
|
|
// Send request
|
|
readPayloadBytes, err := json.Marshal(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to encode read_file request: %w", err)
|
|
}
|
|
msg := Message{
|
|
Type: MsgTypeReadFile,
|
|
ID: req.RequestID,
|
|
Timestamp: time.Now(),
|
|
Payload: readPayloadBytes,
|
|
}
|
|
|
|
ac.writeMu.Lock()
|
|
sendErr := s.sendMessage(ac.conn, msg)
|
|
ac.writeMu.Unlock()
|
|
|
|
if sendErr != nil {
|
|
readLog.Error().
|
|
Err(sendErr).
|
|
Dur("duration", time.Since(startedAt)).
|
|
Msg("Failed to send read_file request to agent")
|
|
return nil, fmt.Errorf("failed to send read_file request: %w", sendErr)
|
|
}
|
|
|
|
// Wait for result
|
|
timeout := readFileTimeout
|
|
timer := time.NewTimer(timeout)
|
|
defer func() {
|
|
if !timer.Stop() {
|
|
select {
|
|
case <-timer.C:
|
|
default:
|
|
}
|
|
}
|
|
}()
|
|
|
|
select {
|
|
case result := <-respCh:
|
|
readLog.Info().
|
|
Bool("success", result.Success).
|
|
Int("exit_code", result.ExitCode).
|
|
Int64("agent_duration_ms", result.Duration).
|
|
Dur("duration", time.Since(startedAt)).
|
|
Msg("Agent read_file completed")
|
|
return &result, nil
|
|
case <-timer.C:
|
|
s.cancelAgentRequest(ac, req.RequestID, "read_file")
|
|
return nil, fmt.Errorf("read_file timed out after %v", timeout)
|
|
case <-ctx.Done():
|
|
s.cancelAgentRequest(ac, req.RequestID, "read_file")
|
|
return nil, fmt.Errorf("read_file %q on agent %q canceled: %w", req.RequestID, agentID, ctx.Err())
|
|
case <-ac.done:
|
|
return nil, fmt.Errorf("agent %s disconnected before read_file result", agentID)
|
|
case <-s.shutdown:
|
|
return nil, errServerShuttingDown
|
|
}
|
|
}
|
|
|
|
// GetConnectedAgents returns a list of currently connected agents
|
|
func (s *Server) GetConnectedAgents() []ConnectedAgent {
|
|
return s.GetConnectedAgentsForOrganization(defaultOrganizationID)
|
|
}
|
|
|
|
// GetConnectedAgentsForOrganization returns only currently admitted sessions
|
|
// owned by one tenant.
|
|
func (s *Server) GetConnectedAgentsForOrganization(organizationID string) []ConnectedAgent {
|
|
if s == nil {
|
|
return nil
|
|
}
|
|
organizationID = normalizeOrganizationID(organizationID)
|
|
s.mu.RLock()
|
|
ids := make([]string, 0, len(s.agents))
|
|
for _, ac := range s.agents {
|
|
if normalizeOrganizationID(ac.admission.OrganizationID) == organizationID {
|
|
ids = append(ids, ac.agent.AgentID)
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
agents := make([]ConnectedAgent, 0, len(ids))
|
|
for _, agentID := range ids {
|
|
if ac, ok := s.connectionForOrganization(organizationID, agentID); ok {
|
|
agents = append(agents, ac.agent)
|
|
}
|
|
}
|
|
return agents
|
|
}
|
|
|
|
// IsAgentConnected checks if an agent is currently connected
|
|
func (s *Server) IsAgentConnected(agentID string) bool {
|
|
return s.IsAgentConnectedForOrganization(defaultOrganizationID, agentID)
|
|
}
|
|
|
|
// IsAgentConnectedForOrganization checks command-channel admission rather than
|
|
// telemetry liveness.
|
|
func (s *Server) IsAgentConnectedForOrganization(organizationID, agentID string) bool {
|
|
_, ok := s.connectionForOrganization(organizationID, agentID)
|
|
return ok
|
|
}
|
|
|
|
// GetAgentForHost finds the agent for a given hostname using the canonical
|
|
// hostname-equivalence contract shared with the unified identity layer.
|
|
func (s *Server) GetAgentForHost(hostname string) (string, bool) {
|
|
return s.GetAgentForHostForOrganization(defaultOrganizationID, hostname)
|
|
}
|
|
|
|
// GetAgentForHostForOrganization resolves a hostname only within one tenant.
|
|
func (s *Server) GetAgentForHostForOrganization(organizationID, hostname string) (string, bool) {
|
|
if s == nil {
|
|
return "", false
|
|
}
|
|
organizationID = normalizeOrganizationID(organizationID)
|
|
s.mu.RLock()
|
|
ids := make([]string, 0, len(s.agents))
|
|
for _, ac := range s.agents {
|
|
if normalizeOrganizationID(ac.admission.OrganizationID) == organizationID &&
|
|
unifiedresources.HostnamesEquivalent(ac.agent.Hostname, hostname) {
|
|
ids = append(ids, ac.agent.AgentID)
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
if len(ids) != 1 {
|
|
return "", false
|
|
}
|
|
for _, agentID := range ids {
|
|
if _, ok := s.connectionForOrganization(organizationID, agentID); ok {
|
|
return agentID, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
// GetAgentForTokenForOrganization resolves the canonical live command session
|
|
// for the same enrollment token that owns a telemetry resource.
|
|
func (s *Server) GetAgentForTokenForOrganization(organizationID, tokenID string) (string, bool) {
|
|
if s == nil || strings.TrimSpace(tokenID) == "" {
|
|
return "", false
|
|
}
|
|
organizationID = normalizeOrganizationID(organizationID)
|
|
tokenID = strings.TrimSpace(tokenID)
|
|
s.mu.RLock()
|
|
ids := make([]string, 0, 1)
|
|
for _, ac := range s.agents {
|
|
if normalizeOrganizationID(ac.admission.OrganizationID) == organizationID &&
|
|
strings.TrimSpace(ac.admission.TokenID) == tokenID {
|
|
ids = append(ids, ac.agent.AgentID)
|
|
}
|
|
}
|
|
s.mu.RUnlock()
|
|
if len(ids) != 1 {
|
|
return "", false
|
|
}
|
|
for _, agentID := range ids {
|
|
if _, ok := s.connectionForOrganization(organizationID, agentID); ok {
|
|
return agentID, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
// GetAgentForIdentityForOrganization resolves a live command session only
|
|
// when one tenant-scoped admission matches both the supplied agent ID and the
|
|
// canonical hostname. This is the safe recovery path for telemetry whose
|
|
// last-seen token ID predates an enrollment-token rotation.
|
|
func (s *Server) GetAgentForIdentityForOrganization(organizationID, agentID, hostname string) (string, bool) {
|
|
if s == nil {
|
|
return "", false
|
|
}
|
|
organizationID = normalizeOrganizationID(organizationID)
|
|
agentID = strings.TrimSpace(agentID)
|
|
hostname = strings.TrimSpace(hostname)
|
|
if agentID == "" || hostname == "" {
|
|
return "", false
|
|
}
|
|
ac, ok := s.connectionForOrganization(organizationID, agentID)
|
|
if !ok || normalizeOrganizationID(ac.admission.OrganizationID) != organizationID ||
|
|
strings.TrimSpace(ac.admission.AgentID) != agentID ||
|
|
!unifiedresources.HostnamesEquivalent(ac.admission.Hostname, hostname) {
|
|
return "", false
|
|
}
|
|
return agentID, true
|
|
}
|
|
|
|
// --- Deploy protocol ---
|
|
|
|
// SubscribeDeployProgress registers a channel to receive deploy progress
|
|
// events for the given agent and job ID. Returns a buffered channel. The caller
|
|
// must call UnsubscribeDeployProgress when done.
|
|
func (s *Server) SubscribeDeployProgress(agentID, jobID string, bufSize int) chan DeployProgressPayload {
|
|
return s.SubscribeDeployProgressForOrganization(defaultOrganizationID, agentID, jobID, bufSize)
|
|
}
|
|
|
|
func (s *Server) SubscribeDeployProgressForOrganization(organizationID, agentID, jobID string, bufSize int) chan DeployProgressPayload {
|
|
if bufSize <= 0 {
|
|
bufSize = 64
|
|
}
|
|
ch := make(chan DeployProgressPayload, bufSize)
|
|
s.mu.Lock()
|
|
s.deploySubs[deploySubKey(agentSessionKey(organizationID, agentID), jobID)] = ch
|
|
s.mu.Unlock()
|
|
return ch
|
|
}
|
|
|
|
// UnsubscribeDeployProgress removes and closes the progress subscriber for an agent's job.
|
|
// Safe to call multiple times — a no-op if already unsubscribed (e.g. by readLoop cleanup).
|
|
func (s *Server) UnsubscribeDeployProgress(agentID, jobID string) {
|
|
s.UnsubscribeDeployProgressForOrganization(defaultOrganizationID, agentID, jobID)
|
|
}
|
|
|
|
func (s *Server) UnsubscribeDeployProgressForOrganization(organizationID, agentID, jobID string) {
|
|
key := deploySubKey(agentSessionKey(organizationID, agentID), jobID)
|
|
s.mu.Lock()
|
|
ch, exists := s.deploySubs[key]
|
|
delete(s.deploySubs, key)
|
|
s.mu.Unlock()
|
|
if exists {
|
|
close(ch)
|
|
}
|
|
}
|
|
|
|
// SendDeployPreflight sends a preflight check command to the source agent.
|
|
// The caller should subscribe to deploy progress for the job ID before calling
|
|
// this method. Results stream back as DeployProgressPayload messages.
|
|
func (s *Server) SendDeployPreflight(ctx context.Context, agentID string, payload DeployPreflightPayload) error {
|
|
payload.RequestID = strings.TrimSpace(payload.RequestID)
|
|
return s.sendDeployCommand(ctx, agentID, MsgTypeDeployPreflight, payload.RequestID, payload)
|
|
}
|
|
|
|
// SendDeployInstall sends an install command to the source agent.
|
|
// The caller should subscribe to deploy progress for the job ID before calling
|
|
// this method. Results stream back as DeployProgressPayload messages.
|
|
func (s *Server) SendDeployInstall(ctx context.Context, agentID string, payload DeployInstallPayload) error {
|
|
payload.RequestID = strings.TrimSpace(payload.RequestID)
|
|
return s.sendDeployCommand(ctx, agentID, MsgTypeDeployInstall, payload.RequestID, payload)
|
|
}
|
|
|
|
// SendDeployCancel sends a cancel command to the source agent.
|
|
func (s *Server) SendDeployCancel(ctx context.Context, agentID string, payload DeployCancelPayload) error {
|
|
payload.RequestID = strings.TrimSpace(payload.RequestID)
|
|
return s.sendDeployCommand(ctx, agentID, MsgTypeDeployCancelJob, payload.RequestID, payload)
|
|
}
|
|
|
|
func (s *Server) sendDeployCommand(ctx context.Context, agentID string, msgType MessageType, requestID string, payload any) error {
|
|
agentID = strings.TrimSpace(agentID)
|
|
if agentID == "" {
|
|
return fmt.Errorf("agent id is required")
|
|
}
|
|
|
|
ac, ok := s.connectionForContext(ctx, agentID)
|
|
if !ok {
|
|
return fmt.Errorf("agent %s not connected", agentID)
|
|
}
|
|
if err := requireLegacyFullTrustConnection(ac, "deploy command"); err != nil {
|
|
return err
|
|
}
|
|
|
|
requestID = strings.TrimSpace(requestID)
|
|
if requestID == "" {
|
|
return fmt.Errorf("request id is required for deploy commands")
|
|
}
|
|
if len(requestID) > maxRequestIDLength {
|
|
return fmt.Errorf("request id exceeds %d characters", maxRequestIDLength)
|
|
}
|
|
|
|
msg, err := NewMessage(msgType, requestID, payload)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to encode deploy command: %w", err)
|
|
}
|
|
|
|
ac.writeMu.Lock()
|
|
err = s.sendMessage(ac.conn, msg)
|
|
ac.writeMu.Unlock()
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("failed to send deploy command: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|