pulse/internal/monitoring/availability_probe_agent.go

package monitoring

import (
	"errors"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/availabilityprobe"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
	pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
	"github.com/rcourtman/pulse-go-rewrite/pkg/tlsutil"
	"github.com/rs/zerolog/log"
)

// availabilityProbeStaleFloor is the shortest window after which a probe-assigned
// target without a fresh report reads as indeterminate.
const availabilityProbeStaleFloor = 5 * time.Minute

// availabilityProbeStaleError is the single read-time explanation shown when an
// assigned agent stops reporting.
const availabilityProbeStaleError = "no recent report from probe agent"

// ProbeAvailabilityResult is one availability observation reported by a remote
// host agent that owns the target's execution.
type ProbeAvailabilityResult struct {
	TargetID      string
	Outcome       availabilityprobe.Outcome
	LatencyMillis int64
	CheckedAt     time.Time
	Error         string
	Certificate   *tlsutil.CertificateObservation
}

// availabilityProbeAssignmentTracker provides a grace reference for a newly
// assigned target before its first report. AgentID is retained so reassigning
// the same target to a different probe starts a fresh grace window.
type availabilityProbeAssignmentTracker struct {
	AgentID string
	Since   time.Time
}

// probeAvailabilityResultsFromReport converts the wire results carried by a
// host agent report. Outcome vocabulary is normalized here so the ingestion
// path never has to trust an agent's spelling; probeResultOutcome then decides
// what an unknown outcome means for failure accounting.
func probeAvailabilityResultsFromReport(reported []agentshost.AvailabilityProbeResult) []ProbeAvailabilityResult {
	if len(reported) == 0 {
		return nil
	}
	results := make([]ProbeAvailabilityResult, 0, len(reported))
	for _, entry := range reported {
		outcome := availabilityprobe.Outcome(strings.ToLower(strings.TrimSpace(entry.Outcome)))
		switch outcome {
		case availabilityprobe.OutcomeReachable, availabilityprobe.OutcomeUnreachable, availabilityprobe.OutcomeIndeterminate:
		default:
			outcome = availabilityprobe.OutcomeIndeterminate
		}
		results = append(results, ProbeAvailabilityResult{
			TargetID:      strings.TrimSpace(entry.TargetID),
			Outcome:       outcome,
			LatencyMillis: entry.LatencyMillis,
			CheckedAt:     entry.CheckedAt,
			Error:         strings.TrimSpace(entry.Error),
			Certificate:   entry.Certificate.Clone(),
		})
	}
	return results
}

// effectiveProbeAgentID returns the host agent that currently owns execution of
// the target. It collapses to local execution ("") whenever the external probe
// entitlement is absent, so a license lapse resumes local polling instead of
// stranding the check on an agent that is no longer allowed to run it.
func (m *Monitor) effectiveProbeAgentID(target config.AvailabilityTarget) string {
	assigned := strings.TrimSpace(target.ProbeAgentID)
	if assigned == "" {
		return ""
	}
	if !m.hasLicensedFeature(pkglicensing.FeatureExternalProbe) {
		return ""
	}
	return assigned
}

// ApplyProbeAvailabilityResults ingests availability results reported by a host
// agent. Results are accepted only for targets currently assigned to that agent.
func (m *Monitor) ApplyProbeAvailabilityResults(hostID string, results []ProbeAvailabilityResult) {
	m.applyProbeAvailabilityResultsAt(hostID, results, time.Now().UTC())
}

// applyProbeAvailabilityResultsAt keeps server receipt time explicit for the
// host-report path and deterministic tests. Agent-authored CheckedAt describes
// the observation; receivedAt is the authoritative signal that the probe is
// still reporting.
func (m *Monitor) applyProbeAvailabilityResultsAt(hostID string, results []ProbeAvailabilityResult, receivedAt time.Time) {
	if m == nil {
		return
	}
	hostID = strings.TrimSpace(hostID)
	if hostID == "" || len(results) == 0 {
		return
	}
	if receivedAt.IsZero() {
		receivedAt = time.Now().UTC()
	} else {
		receivedAt = receivedAt.UTC()
	}

	applied := 0
	for _, result := range results {
		targetID := strings.TrimSpace(result.TargetID)
		if targetID == "" {
			continue
		}
		target, ok := m.availabilityTargetByID(targetID)
		if !ok {
			log.Debug().
				Str("hostID", hostID).
				Str("targetID", targetID).
				Msg("Rejecting probe availability result for unknown target")
			continue
		}
		if m.effectiveProbeAgentID(target) != hostID {
			log.Debug().
				Str("hostID", hostID).
				Str("targetID", targetID).
				Msg("Rejecting probe availability result from an agent that does not own the target")
			continue
		}

		checkedAt := result.CheckedAt
		if checkedAt.IsZero() {
			checkedAt = receivedAt
		}
		latency := time.Duration(result.LatencyMillis) * time.Millisecond
		if latency < 0 {
			latency = 0
		}
		outcome, probeErr := probeResultOutcome(result)
		m.applyAvailabilityObservation(target, checkedAt.UTC(), latency, outcome, probeErr, result.Certificate, hostID, receivedAt)
		applied++
	}

	if applied == 0 {
		return
	}
	m.updateResourceStore(m.GetState())
}

// probeResultOutcome normalizes a reported outcome and derives the failure
// signal. An unreachable report without a message still has to fail, otherwise
// remote checks would never accumulate consecutive failures.
func probeResultOutcome(result ProbeAvailabilityResult) (AvailabilityProbeOutcome, error) {
	outcome := AvailabilityProbeOutcome(strings.ToLower(strings.TrimSpace(string(result.Outcome))))
	message := strings.TrimSpace(result.Error)
	switch outcome {
	case AvailabilityProbeReachable, AvailabilityProbeUnreachable, AvailabilityProbeIndeterminate:
	default:
		if message != "" {
			outcome = AvailabilityProbeUnreachable
		} else {
			outcome = AvailabilityProbeIndeterminate
		}
	}
	if outcome == AvailabilityProbeUnreachable {
		if message == "" {
			message = "probe agent reported the target unreachable"
		}
		return outcome, errors.New(message)
	}
	if message != "" {
		return AvailabilityProbeUnreachable, errors.New(message)
	}
	return outcome, nil
}

// deriveAvailabilityProbeStaleness reports the status a reader should see for a
// probe-assigned target. Stored state is never mutated: an agent that stops
// reporting must read as indeterminate without erasing its last observation.
func (m *Monitor) deriveAvailabilityProbeStaleness(
	target config.AvailabilityTarget,
	status AvailabilityProbeStatus,
	now time.Time,
) AvailabilityProbeStatus {
	agentID := m.effectiveProbeAgentID(target)
	if agentID == "" {
		return status
	}
	reportingAgentID := strings.TrimSpace(status.ProbeAgentID)
	if reportingAgentID != agentID {
		status = availabilityStatusFromTarget(target)
	}
	status.ProbeAgentID = agentID
	reference := status.FreshnessTime()
	if reference.IsZero() {
		reference = m.availabilityProbeAssignmentReference(target.ID, agentID, now)
	}
	if !availabilityProbeReportIsStale(target, reference, now) {
		return status
	}
	status.Outcome = string(AvailabilityProbeIndeterminate)
	status.Available = false
	status.LastError = availabilityProbeStaleError
	status.LatencyMillis = 0
	return status
}

func (m *Monitor) availabilityProbeAssignmentReference(targetID, agentID string, now time.Time) time.Time {
	if m == nil {
		return now
	}
	targetID = strings.TrimSpace(targetID)
	agentID = strings.TrimSpace(agentID)
	if targetID == "" || agentID == "" {
		return now
	}

	m.mu.Lock()
	defer m.mu.Unlock()
	if m.availabilityProbeTrackers == nil {
		m.availabilityProbeTrackers = make(map[string]availabilityProbeAssignmentTracker)
	}
	if tracker, ok := m.availabilityProbeTrackers[targetID]; ok && tracker.AgentID == agentID && !tracker.Since.IsZero() {
		return tracker.Since
	}
	m.availabilityProbeTrackers[targetID] = availabilityProbeAssignmentTracker{
		AgentID: agentID,
		Since:   now,
	}
	return now
}

func availabilityProbeStatusIsStale(status AvailabilityProbeStatus) bool {
	return status.Outcome == string(AvailabilityProbeIndeterminate) &&
		strings.EqualFold(strings.TrimSpace(status.LastError), availabilityProbeStaleError)
}

func availabilityProbeReportIsStale(target config.AvailabilityTarget, lastChecked time.Time, now time.Time) bool {
	if lastChecked.IsZero() {
		return true
	}
	return now.Sub(lastChecked) > availabilityProbeStaleWindow(target)
}

func availabilityProbeStaleWindow(target config.AvailabilityTarget) time.Duration {
	window := time.Duration(target.EffectivePollIntervalSecs()) * 3 * time.Second
	if window < availabilityProbeStaleFloor {
		return availabilityProbeStaleFloor
	}
	return window
}

// availabilityProbeTargetsForAgent returns the probe payload for the targets the
// given agent currently owns.
func (m *Monitor) availabilityProbeTargetsForAgent(hostID string) []map[string]interface{} {
	hostID = strings.TrimSpace(hostID)
	if hostID == "" {
		return nil
	}
	var assigned []map[string]interface{}
	for _, target := range m.availabilityTargets() {
		if m.effectiveProbeAgentID(target) != hostID {
			continue
		}
		assigned = append(assigned, availabilityProbeAgentTargetPayload(target))
	}
	return assigned
}

// availabilityProbeAgentTargetPayload carries only what the agent needs to run
// the check. Failure accounting and resource linkage stay server-side.
func availabilityProbeAgentTargetPayload(target config.AvailabilityTarget) map[string]interface{} {
	payload := map[string]interface{}{
		"id":                  target.ID,
		"name":                target.DisplayName(),
		"targetKind":          string(target.TargetKind),
		"address":             target.Address,
		"protocol":            string(target.Protocol),
		"enabled":             target.Enabled,
		"pollIntervalSeconds": target.EffectivePollIntervalSecs(),
		"timeoutMillis":       target.EffectiveTimeoutMillis(),
	}
	if target.Port > 0 {
		payload["port"] = target.Port
	}
	if path := strings.TrimSpace(target.Path); path != "" {
		payload["path"] = path
	}
	if target.UDPMode != "" {
		payload["udpMode"] = string(target.UDPMode)
	}
	if target.UDPRequest != "" {
		payload["udpRequest"] = target.UDPRequest
	}
	if target.UDPExpected != "" {
		payload["udpExpectedResponse"] = target.UDPExpected
	}
	return payload
}