From 0ff0a37f8d307c215cfde1a67599cdd2cc044533 Mon Sep 17 00:00:00 2001 From: "courtmanr@gmail.com" Date: Wed, 29 Jul 2026 21:27:18 +0100 Subject: [PATCH] Use server time for probe freshness --- docs/CONFIGURATION.md | 12 +-- docs/UNIFIED_AGENT.md | 14 +-- .../v6/internal/subsystems/agent-lifecycle.md | 6 ++ .../v6/internal/subsystems/api-contracts.md | 10 +++ .../v6/internal/subsystems/monitoring.md | 36 +++++--- .../internal/subsystems/storage-recovery.md | 6 +- frontend-modern/public/docs/CONFIGURATION.md | 12 +-- internal/api/connections_aggregator.go | 9 +- internal/api/connections_aggregator_test.go | 34 +++++++ internal/monitoring/availability_poller.go | 59 +++++++++++-- .../monitoring/availability_probe_agent.go | 27 ++++-- .../availability_probe_agent_test.go | 88 +++++++++++++++---- internal/monitoring/availability_udp_test.go | 2 +- 13 files changed, 254 insertions(+), 61 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 925d68d98..1a7e95271 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -509,11 +509,13 @@ its signed agent configuration, runs it on the configured interval, and reports results back with its reports; results are only accepted from the currently assigned agent. An assigned check is not also run locally. If no report arrives for several intervals the check shows as indeterminate -("no recent report from probe agent"). After at least five minutes without a -report, Pulse raises one warning per disconnected probe through the normal -notification routes, even when that probe owns several checks. The incident is -keyed to the probe agent rather than an arbitrary check, so changing assignments -does not reopen it. If the agent heartbeat is also offline, the existing +("no recent report from probe agent"). Freshness uses the Pulse server's receipt +time, so clock drift on the probe cannot create or hide a disconnect. After at +least five minutes without a report, Pulse raises one warning per disconnected +probe through the normal notification routes, even when that probe owns several +checks. The incident is keyed to the probe agent rather than an arbitrary check, +so changing assignments does not reopen it. If the agent heartbeat is also +offline, the existing host-offline alert owns the incident instead of producing a duplicate probe warning. A paired Pulse Mobile client receives a privacy-safe `external_probe_offline` Relay push linked to that canonical alert. If the whole diff --git a/docs/UNIFIED_AGENT.md b/docs/UNIFIED_AGENT.md index a075a8c1c..9c49617f2 100644 --- a/docs/UNIFIED_AGENT.md +++ b/docs/UNIFIED_AGENT.md @@ -106,12 +106,14 @@ in-memory queue; if the agent cannot deliver for several check intervals the check shows as indeterminate in Pulse until reports resume. After the five-minute minimum grace window, Pulse raises one `availability_probe_unavailable` warning per disconnected probe, regardless of -how many checks it owns. That warning uses the normal email, webhook, Apprise, -and recovery-notification pipeline. When Pulse Mobile is paired through Relay, -Pulse also sends a privacy-safe `external_probe_offline` push linked to the -canonical mobile attention item without exposing target names or addresses. -The alert identity belongs to the probe agent, so adding or removing an -assigned check does not resolve and reopen it. +how many checks it owns. Pulse measures that reporting window from server receipt +time rather than the agent's clock, so clock skew cannot create or conceal the +disconnect. That warning uses the normal email, webhook, Apprise, and +recovery-notification pipeline. When Pulse Mobile is paired through Relay, Pulse +also sends a privacy-safe `external_probe_offline` push linked to the canonical +mobile attention item without exposing target names or addresses. The alert +identity belongs to the probe agent, so adding or removing an assigned check +does not resolve and reopen it. When the host heartbeat itself is offline, Pulse keeps the existing host-offline alert as the single canonical incident and suppresses the diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 33394ab3f..91663ccca 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -60,6 +60,12 @@ and reassignment start a fresh reporting grace window; a result from the previous agent must not satisfy the new lease. Once the host heartbeat itself is unhealthy, the canonical host-offline lifecycle owns the outage so the availability-probe lifecycle cannot create a duplicate whole-agent incident. +The adjacent `/api/connections` availability row uses monitoring's +server-receipt freshness reference for that lease while preserving the +agent-authored check time as observation metadata. That receipt timestamp is +probe-result liveness only: lifecycle consumers must not reinterpret it as a +host heartbeat, enrollment renewal, command-channel readiness, or fleet-update +observation. ## Canonical Files diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 325821c85..bdc072db2 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -1212,6 +1212,11 @@ payload shape change when the portal presents compact client rows. `availability` must route management handoffs, pause, remove, and test actions to those availability-target endpoints and must not reuse node, SSH, platform API, or Pulse Agent setup payloads. + For probe-assigned rows, the Connections ledger consumes monitoring's + canonical server-receipt freshness reference and its already-derived stale + verdict. It must not independently age the agent-authored `LastSuccess` + timestamp, because agent clock skew could make Connections disagree with + the `network-endpoint` resource and canonical probe alert lifecycle. Mock mode must expose authored availability targets through those same list, saved-test, and connections-ledger payloads so demo endpoints exercise the canonical API contract rather than a frontend-only fixture. @@ -7846,6 +7851,11 @@ as a `network-endpoint`. Browser callers may test unsaved or saved targets, but the persisted target list remains owned by `/api/availability-targets` and must be managed from `/settings/monitoring/availability`, not reconstructed from resource snapshots or monitored-system counts. +Remote probe rows preserve the agent-authored check time as observation +metadata, but API freshness and `lastSeenAt` projections use the server-authored +receipt time supplied by monitoring. API consumers must not substitute the +agent clock for disconnect detection or apply a second, shorter generic +Connections staleness window. Mock availability fixtures must still behave like saved targets: `/api/connections` reports them as availability rows, `/api/availability-targets` lists them with probe status, and saved-test calls return the synthetic probe result instead of diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 91041fe84..aae6fa289 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -324,13 +324,16 @@ yields no entry and must say so in the log rather than passing silently, and it must never prevent the remaining configured hosts from being suppressed. External availability-probe freshness is evaluated against the effective -target cadence with a five-minute minimum grace. Missing, stale, or -wrong-agent results are indeterminate monitoring evidence: their -`network-endpoint` resources degrade to warning without manufacturing a target -reachability incident. Monitoring aggregates stale targets by current agent, -updates the single canonical probe alert lifecycle, and treats a fresh result -from that same assignment as recovery. Assignment trackers are removed with -their targets and reset when agent identity changes. +target cadence with a five-minute minimum grace. The server-authored receipt +time is authoritative for reporting freshness; the agent-authored check time +remains observation metadata and must not create or suppress a disconnect when +clocks differ. Missing, stale, or wrong-agent results are indeterminate +monitoring evidence: their `network-endpoint` resources degrade to warning +without manufacturing a target reachability incident. Monitoring aggregates +stale targets by current agent, updates the single canonical probe alert +lifecycle, and treats a fresh result from that same assignment as recovery. +Assignment trackers are removed with their targets and reset when agent identity +changes. ## Canonical Files @@ -999,9 +1002,12 @@ to resolve. Every completed probe also authors an operational-trust `EvidenceEnvelope` with provider `availability`, collector `availability-poller`, the saved target as its provider reference, the exact observation/ingest times, and a validity window of twice the effective polling -interval. Before the first completed probe, evidence is explicitly partial and -unknown with reason `availability_not_observed`; monitoring must never encode -that state as a confirmed failure or a healthy observation. The registry owns +interval for local checks. Remote-probe evidence instead uses its canonical +server-receipt freshness window (three effective intervals with the five-minute +minimum), so resource evidence, Connections state, and the probe alert lifecycle +cannot disagree. Before the first completed probe, evidence is explicitly +partial and unknown with reason `availability_not_observed`; monitoring must +never encode that state as a confirmed failure or a healthy observation. The registry owns binding the source envelope to the check resource and cloning a separately bound envelope for any matched-resource facet projection after correlation. Availability target kind is monitoring-owned runtime metadata, not a frontend @@ -1029,10 +1035,12 @@ local and the normal poll provider resumes the target on its next planning cycle, without a restart. Reported results are accepted only from the agent that currently owns the target, and results for any other target or from any other agent are dropped. Failure accounting, thresholds, and incident projection stay -server-side. When an assigned agent stops reporting, monitoring derives -indeterminate with a stale-report explanation at read time through the shared -probe-status snapshot rather than mutating stored state, so every availability -consumer sees the same staleness verdict. +server-side. The agent-authored observation time remains visible as the target's +last check, but staleness uses server receipt time so slow or fast agent clocks +cannot manufacture or conceal a disconnect. When an assigned agent stops +reporting, monitoring derives indeterminate with a stale-report explanation at +read time through the shared probe-status snapshot rather than mutating stored +state, so every availability consumer sees the same staleness verdict. Mock-mode Discovery context follows the same fixture-graph rule. Demo service details such as detected version, config/data/log paths, Docker bind mounts, ports, and suggested web URLs may be authored in mock fixtures, but consumers diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 813476d99..4a62aba3b 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -84,7 +84,11 @@ indeterminate reachability evidence. Even when its target is a NAS, PBS, datastore, or other storage resource, the resulting warning and canonical probe alert are not backup failure, protection drift, snapshot damage, recovery-point loss, or restore-readiness evidence. Storage/recovery consumers -must wait for their domain-owned observations. +must wait for their domain-owned observations. The shared Connections row and +availability evidence use server receipt time for remote-probe freshness while +retaining agent check time as observation metadata; storage/recovery must not +reinterpret either timestamp as backup recency, datastore health, or recovery +freshness. ## Canonical Files diff --git a/frontend-modern/public/docs/CONFIGURATION.md b/frontend-modern/public/docs/CONFIGURATION.md index 925d68d98..1a7e95271 100644 --- a/frontend-modern/public/docs/CONFIGURATION.md +++ b/frontend-modern/public/docs/CONFIGURATION.md @@ -509,11 +509,13 @@ its signed agent configuration, runs it on the configured interval, and reports results back with its reports; results are only accepted from the currently assigned agent. An assigned check is not also run locally. If no report arrives for several intervals the check shows as indeterminate -("no recent report from probe agent"). After at least five minutes without a -report, Pulse raises one warning per disconnected probe through the normal -notification routes, even when that probe owns several checks. The incident is -keyed to the probe agent rather than an arbitrary check, so changing assignments -does not reopen it. If the agent heartbeat is also offline, the existing +("no recent report from probe agent"). Freshness uses the Pulse server's receipt +time, so clock drift on the probe cannot create or hide a disconnect. After at +least five minutes without a report, Pulse raises one warning per disconnected +probe through the normal notification routes, even when that probe owns several +checks. The incident is keyed to the probe agent rather than an arbitrary check, +so changing assignments does not reopen it. If the agent heartbeat is also +offline, the existing host-offline alert owns the incident instead of producing a duplicate probe warning. A paired Pulse Mobile client receives a privacy-safe `external_probe_offline` Relay push linked to that canonical alert. If the whole diff --git a/internal/api/connections_aggregator.go b/internal/api/connections_aggregator.go index 0a4d36ec6..055631e8e 100644 --- a/internal/api/connections_aggregator.go +++ b/internal/api/connections_aggregator.go @@ -1366,6 +1366,9 @@ func deriveAvailabilityConnectionState(target config.AvailabilityTarget, status var lastSeen *time.Time if !status.LastSuccess.IsZero() { t := status.LastSuccess + if freshness := status.FreshnessTime(); strings.TrimSpace(status.ProbeAgentID) != "" && !freshness.IsZero() { + t = freshness + } lastSeen = &t } @@ -1392,7 +1395,11 @@ func deriveAvailabilityConnectionState(target config.AvailabilityTarget, status } return ConnectionStateUnreachable, reason, lastSeen, lastError } - if lastSeen != nil { + // Probe-assigned statuses have already passed through monitoring's + // canonical receipt-time staleness derivation. Do not apply this generic + // LastSuccess clock a second time or the Connections projection can + // disagree with the resource and alert lifecycle. + if lastSeen != nil && strings.TrimSpace(status.ProbeAgentID) == "" { staleThreshold := time.Duration(target.EffectivePollIntervalSecs()*2) * time.Second if staleThreshold < connectionStaleThreshold { staleThreshold = connectionStaleThreshold diff --git a/internal/api/connections_aggregator_test.go b/internal/api/connections_aggregator_test.go index 717fec3c5..b30f4ab46 100644 --- a/internal/api/connections_aggregator_test.go +++ b/internal/api/connections_aggregator_test.go @@ -239,6 +239,40 @@ func TestBuildConnections_AvailabilityTargetStateFromProbeStatus(t *testing.T) { } } +func TestDeriveAvailabilityConnectionStateUsesCanonicalRemoteProbeFreshness(t *testing.T) { + receivedAt := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC) + agentCheckedAt := receivedAt.Add(-24 * time.Hour) + target := config.NormalizeAvailabilityTarget(config.AvailabilityTarget{ + ID: "remote-sensor", + Name: "Remote sensor", + Address: "sensor.example", + Protocol: config.AvailabilityProbeICMP, + Enabled: true, + PollIntervalSecs: 60, + ProbeAgentID: "probe-agent", + }) + status := monitoring.AvailabilityProbeStatus{ + TargetID: target.ID, + Available: true, + LastChecked: agentCheckedAt, + LastSuccess: agentCheckedAt, + ProbeAgentID: "probe-agent", + ProbeReportReceivedAt: receivedAt, + } + + state, reason, lastSeen, lastError := deriveAvailabilityConnectionState( + target, + status, + receivedAt.Add(4*time.Minute), + ) + if state != ConnectionStateActive || reason != "" || lastError != nil { + t.Fatalf("connection = state %q reason %q error %+v, want active", state, reason, lastError) + } + if lastSeen == nil || !lastSeen.Equal(receivedAt) { + t.Fatalf("last seen = %v, want server receipt %v", lastSeen, receivedAt) + } +} + func TestBuildConnections_AgentHostAliasesIncludeReportedIdentityHints(t *testing.T) { now := time.Now() in := aggregatorInputs{ diff --git a/internal/monitoring/availability_poller.go b/internal/monitoring/availability_poller.go index d248ed9e7..0d906ca87 100644 --- a/internal/monitoring/availability_poller.go +++ b/internal/monitoring/availability_poller.go @@ -33,6 +33,21 @@ type AvailabilityProbeStatus struct { LastError string `json:"lastError,omitempty"` FailureThreshold int `json:"failureThreshold,omitempty"` ProbeAgentID string `json:"probeAgentId,omitempty"` + // ProbeReportReceivedAt is server-authored freshness evidence for a remote + // observation. Keep it off the wire: LastChecked remains the agent's + // observation time, while disconnect detection must not trust agent clock + // skew in either direction. + ProbeReportReceivedAt time.Time `json:"-"` +} + +// FreshnessTime returns the authoritative liveness reference for this status. +// Remote observations use server receipt time; local observations use their +// server-authored check time. +func (s AvailabilityProbeStatus) FreshnessTime() time.Time { + if strings.TrimSpace(s.ProbeAgentID) != "" && !s.ProbeReportReceivedAt.IsZero() { + return s.ProbeReportReceivedAt + } + return s.LastChecked } // AvailabilityProbeOutcome and its values are aliases for the shared probe @@ -308,7 +323,7 @@ func (m *Monitor) pollAvailabilityTarget(ctx context.Context, target config.Avai outcome, err := ProbeAvailabilityTargetResult(ctx, target) latency := time.Since(start) checkedAt := time.Now().UTC() - m.applyAvailabilityObservation(target, checkedAt, latency, outcome, err, "") + m.applyAvailabilityObservation(target, checkedAt, latency, outcome, err, "", time.Time{}) m.updateResourceStore(m.GetState()) } @@ -322,8 +337,9 @@ func (m *Monitor) applyAvailabilityObservation( outcome AvailabilityProbeOutcome, probeErr error, probeAgentID string, + probeReportReceivedAt time.Time, ) { - m.setAvailabilityStatus(target, checkedAt, latency, outcome, probeErr, probeAgentID) + m.setAvailabilityStatus(target, checkedAt, latency, outcome, probeErr, probeAgentID, probeReportReceivedAt) if probeErr == nil { if m.stalenessTracker != nil { @@ -339,7 +355,15 @@ func (m *Monitor) applyAvailabilityObservation( m.recordTaskResult(InstanceTypeAvailability, target.ID, nil) } -func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checkedAt time.Time, latency time.Duration, outcome AvailabilityProbeOutcome, probeErr error, probeAgentID string) { +func (m *Monitor) setAvailabilityStatus( + target config.AvailabilityTarget, + checkedAt time.Time, + latency time.Duration, + outcome AvailabilityProbeOutcome, + probeErr error, + probeAgentID string, + probeReportReceivedAt time.Time, +) { if m == nil { return } @@ -347,6 +371,9 @@ func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checke status.Outcome = string(outcome) status.LastChecked = checkedAt status.ProbeAgentID = strings.TrimSpace(probeAgentID) + if status.ProbeAgentID != "" { + status.ProbeReportReceivedAt = probeReportReceivedAt.UTC() + } latencyMs := latency.Milliseconds() if probeErr == nil && latencyMs == 0 { latencyMs = 1 @@ -414,7 +441,7 @@ func availabilityStatusFromTarget(target config.AvailabilityTarget) Availability } func availabilityResourceFromTarget(target config.AvailabilityTarget, status AvailabilityProbeStatus, _ string, now time.Time) (unifiedresources.Resource, unifiedresources.ResourceIdentity) { - lastSeen := status.LastChecked + lastSeen := status.FreshnessTime() if lastSeen.IsZero() { lastSeen = now } @@ -442,7 +469,19 @@ func availabilityResourceFromTarget(target config.AvailabilityTarget, status Ava PollIntervalSeconds: target.EffectivePollIntervalSecs(), TimeoutMillis: target.EffectiveTimeoutMillis(), } - data.Evidence = availabilityEvidenceEnvelope(target, status, lastSeen, now) + observedAt := status.LastChecked + if observedAt.IsZero() { + observedAt = lastSeen + } + if observedAt.After(lastSeen) { + // A fast remote clock cannot author evidence in the server's future. + observedAt = lastSeen + } + ingestedAt := now + if strings.TrimSpace(status.ProbeAgentID) != "" && !status.ProbeReportReceivedAt.IsZero() { + ingestedAt = status.ProbeReportReceivedAt + } + data.Evidence = availabilityEvidenceEnvelope(target, status, observedAt, ingestedAt) resource := unifiedresources.Resource{ Type: unifiedresources.ResourceTypeNetworkEndpoint, Technology: string(target.Protocol), @@ -498,9 +537,13 @@ func availabilityEvidenceEnvelope( return nil } - validUntil := observedAt.Add( - time.Duration(target.EffectivePollIntervalSecs()*2) * time.Second, - ) + freshnessAt := observedAt + validityWindow := time.Duration(target.EffectivePollIntervalSecs()*2) * time.Second + if strings.TrimSpace(status.ProbeAgentID) != "" && !status.ProbeReportReceivedAt.IsZero() { + freshnessAt = status.ProbeReportReceivedAt + validityWindow = availabilityProbeStaleWindow(target) + } + validUntil := freshnessAt.Add(validityWindow) completeness := operationaltrust.EvidenceComplete confidence := operationaltrust.EvidenceConfirmed var reason *operationaltrust.EvidenceReason diff --git a/internal/monitoring/availability_probe_agent.go b/internal/monitoring/availability_probe_agent.go index b8b2a977c..a7aa9bbd8 100644 --- a/internal/monitoring/availability_probe_agent.go +++ b/internal/monitoring/availability_probe_agent.go @@ -83,6 +83,14 @@ func (m *Monitor) effectiveProbeAgentID(target config.AvailabilityTarget) string // ApplyProbeAvailabilityResults ingests availability results reported by a host // agent. Results are accepted only for targets currently assigned to that agent. func (m *Monitor) ApplyProbeAvailabilityResults(hostID string, results []ProbeAvailabilityResult) { + m.applyProbeAvailabilityResultsAt(hostID, results, time.Now().UTC()) +} + +// applyProbeAvailabilityResultsAt keeps server receipt time explicit for the +// host-report path and deterministic tests. Agent-authored CheckedAt describes +// the observation; receivedAt is the authoritative signal that the probe is +// still reporting. +func (m *Monitor) applyProbeAvailabilityResultsAt(hostID string, results []ProbeAvailabilityResult, receivedAt time.Time) { if m == nil { return } @@ -90,6 +98,11 @@ func (m *Monitor) ApplyProbeAvailabilityResults(hostID string, results []ProbeAv if hostID == "" || len(results) == 0 { return } + if receivedAt.IsZero() { + receivedAt = time.Now().UTC() + } else { + receivedAt = receivedAt.UTC() + } applied := 0 for _, result := range results { @@ -115,14 +128,14 @@ func (m *Monitor) ApplyProbeAvailabilityResults(hostID string, results []ProbeAv checkedAt := result.CheckedAt if checkedAt.IsZero() { - checkedAt = time.Now() + checkedAt = receivedAt } latency := time.Duration(result.LatencyMillis) * time.Millisecond if latency < 0 { latency = 0 } outcome, probeErr := probeResultOutcome(result) - m.applyAvailabilityObservation(target, checkedAt.UTC(), latency, outcome, probeErr, hostID) + m.applyAvailabilityObservation(target, checkedAt.UTC(), latency, outcome, probeErr, hostID, receivedAt) applied++ } @@ -176,7 +189,7 @@ func (m *Monitor) deriveAvailabilityProbeStaleness( status = availabilityStatusFromTarget(target) } status.ProbeAgentID = agentID - reference := status.LastChecked + reference := status.FreshnessTime() if reference.IsZero() { reference = m.availabilityProbeAssignmentReference(target.ID, agentID, now) } @@ -224,11 +237,15 @@ func availabilityProbeReportIsStale(target config.AvailabilityTarget, lastChecke if lastChecked.IsZero() { return true } + return now.Sub(lastChecked) > availabilityProbeStaleWindow(target) +} + +func availabilityProbeStaleWindow(target config.AvailabilityTarget) time.Duration { window := time.Duration(target.EffectivePollIntervalSecs()) * 3 * time.Second if window < availabilityProbeStaleFloor { - window = availabilityProbeStaleFloor + return availabilityProbeStaleFloor } - return now.Sub(lastChecked) > window + return window } // availabilityProbeTargetsForAgent returns the probe payload for the targets the diff --git a/internal/monitoring/availability_probe_agent_test.go b/internal/monitoring/availability_probe_agent_test.go index 1d94e2b80..ff96a4264 100644 --- a/internal/monitoring/availability_probe_agent_test.go +++ b/internal/monitoring/availability_probe_agent_test.go @@ -337,9 +337,9 @@ func TestAvailabilityProbeStalenessDerivesIndeterminateAtReadTime(t *testing.T) monitor.SetLicenseChecker(licenseWithExternalProbe(true)) checkedAt := time.Now().UTC() - monitor.ApplyProbeAvailabilityResults("agent-1", []ProbeAvailabilityResult{ + monitor.applyProbeAvailabilityResultsAt("agent-1", []ProbeAvailabilityResult{ {TargetID: "remote", Outcome: AvailabilityProbeReachable, LatencyMillis: 9, CheckedAt: checkedAt}, - }) + }, checkedAt) // Effective poll interval is 60s, so the floor of five minutes governs. fresh := monitor.availabilityStatusSnapshotForTargets([]config.AvailabilityTarget{target}, checkedAt.Add(5*time.Minute)) @@ -382,13 +382,69 @@ func TestAvailabilityProbeStalenessDerivesIndeterminateAtReadTime(t *testing.T) } } +func TestAvailabilityProbeStalenessUsesServerReceiptTimeAcrossAgentClockSkew(t *testing.T) { + target := config.NormalizeAvailabilityTarget(probeAgentTarget("remote", "agent-1")) + monitor := newProbeAgentTestMonitor(t, target) + monitor.SetLicenseChecker(licenseWithExternalProbe(true)) + + receivedAt := time.Now().UTC() + for _, test := range []struct { + name string + checkedAt time.Time + }{ + {name: "slow agent clock", checkedAt: receivedAt.Add(-24 * time.Hour)}, + {name: "fast agent clock", checkedAt: receivedAt.Add(24 * time.Hour)}, + } { + t.Run(test.name, func(t *testing.T) { + monitor.applyProbeAvailabilityResultsAt("agent-1", []ProbeAvailabilityResult{ + {TargetID: "remote", Outcome: AvailabilityProbeReachable, CheckedAt: test.checkedAt}, + }, receivedAt) + + fresh := monitor.availabilityStatusSnapshotForTargets( + []config.AvailabilityTarget{target}, + receivedAt.Add(availabilityProbeStaleFloor), + )["remote"] + if !fresh.Available || availabilityProbeStatusIsStale(fresh) { + t.Fatalf("boundary status = %+v, server receipt must keep the report fresh", fresh) + } + if !fresh.LastChecked.Equal(test.checkedAt) { + t.Fatalf("last checked = %v, want agent observation time %v preserved", fresh.LastChecked, test.checkedAt) + } + resource, _ := availabilityResourceFromTarget(target, fresh, "", receivedAt) + if !resource.LastSeen.Equal(receivedAt) { + t.Fatalf("resource last seen = %v, want server receipt %v", resource.LastSeen, receivedAt) + } + evidence := resource.Availability.Evidence + if evidence == nil || !evidence.IngestedAt.Equal(receivedAt) { + t.Fatalf("evidence = %+v, want server receipt as ingest time", evidence) + } + wantValidUntil := receivedAt.Add(availabilityProbeStaleWindow(target)) + if evidence.ValidUntil == nil || !evidence.ValidUntil.Equal(wantValidUntil) { + t.Fatalf("evidence valid until = %v, want %v", evidence.ValidUntil, wantValidUntil) + } + if evidence.ObservedAt.After(evidence.IngestedAt) { + t.Fatalf("evidence chronology = observed %v after ingested %v", evidence.ObservedAt, evidence.IngestedAt) + } + + stale := monitor.availabilityStatusSnapshotForTargets( + []config.AvailabilityTarget{target}, + receivedAt.Add(availabilityProbeStaleFloor+time.Second), + )["remote"] + if !availabilityProbeStatusIsStale(stale) { + t.Fatalf("post-window status = %+v, want stale by server receipt time", stale) + } + }) + } +} + func TestAvailabilitySupplementalRecordsPresentStaleProbeAsIndeterminate(t *testing.T) { monitor := newProbeAgentTestMonitor(t, probeAgentTarget("remote", "agent-1")) monitor.SetLicenseChecker(licenseWithExternalProbe(true)) - monitor.ApplyProbeAvailabilityResults("agent-1", []ProbeAvailabilityResult{ - {TargetID: "remote", Outcome: AvailabilityProbeReachable, LatencyMillis: 9, CheckedAt: time.Now().UTC().Add(-time.Hour)}, - }) + old := time.Now().UTC().Add(-time.Hour) + monitor.applyProbeAvailabilityResultsAt("agent-1", []ProbeAvailabilityResult{ + {TargetID: "remote", Outcome: AvailabilityProbeReachable, LatencyMillis: 9, CheckedAt: old}, + }, old) records := availabilityPollProvider{}.SupplementalRecords(monitor, "org-a") if len(records) != 1 { @@ -487,13 +543,13 @@ func TestAvailabilityProbeStaleIncidentDeduplicatesPerAgent(t *testing.T) { t.Cleanup(alertManager.Stop) monitor.alertManager = alertManager old := time.Now().UTC().Add(-time.Hour) - monitor.ApplyProbeAvailabilityResults("agent-1", []ProbeAvailabilityResult{ + monitor.applyProbeAvailabilityResultsAt("agent-1", []ProbeAvailabilityResult{ {TargetID: "remote-a", Outcome: AvailabilityProbeReachable, CheckedAt: old}, {TargetID: "remote-b", Outcome: AvailabilityProbeReachable, CheckedAt: old}, - }) - monitor.ApplyProbeAvailabilityResults("agent-2", []ProbeAvailabilityResult{ + }, old) + monitor.applyProbeAvailabilityResultsAt("agent-2", []ProbeAvailabilityResult{ {TargetID: "remote-c", Outcome: AvailabilityProbeReachable, CheckedAt: old}, - }) + }, old) _ = (availabilityPollProvider{}).SupplementalRecords(monitor, "") alertsByAgent := make(map[string]int) @@ -515,9 +571,10 @@ func TestAvailabilityProbeFreshReportResolvesStaleIncident(t *testing.T) { alertManager := alerts.NewManagerWithDataDir(t.TempDir()) t.Cleanup(alertManager.Stop) monitor.alertManager = alertManager - monitor.ApplyProbeAvailabilityResults("agent-1", []ProbeAvailabilityResult{ - {TargetID: "remote", Outcome: AvailabilityProbeReachable, CheckedAt: time.Now().UTC().Add(-time.Hour)}, - }) + old := time.Now().UTC().Add(-time.Hour) + monitor.applyProbeAvailabilityResultsAt("agent-1", []ProbeAvailabilityResult{ + {TargetID: "remote", Outcome: AvailabilityProbeReachable, CheckedAt: old}, + }, old) _ = (availabilityPollProvider{}).SupplementalRecords(monitor, "") if got := alertManager.GetActiveAlerts(); len(got) != 1 || got[0].Type != alerts.ExternalProbeUnavailableAlertType { t.Fatalf("stale alerts = %+v, want one before recovery", got) @@ -547,9 +604,10 @@ func TestAvailabilityProbeHostOfflineOwnsConnectivityLifecycle(t *testing.T) { LastSeen: time.Now().UTC().Add(-time.Hour), IntervalSeconds: 30, }) - monitor.ApplyProbeAvailabilityResults("agent-1", []ProbeAvailabilityResult{ - {TargetID: "remote", Outcome: AvailabilityProbeReachable, CheckedAt: time.Now().UTC().Add(-time.Hour)}, - }) + old := time.Now().UTC().Add(-time.Hour) + monitor.applyProbeAvailabilityResultsAt("agent-1", []ProbeAvailabilityResult{ + {TargetID: "remote", Outcome: AvailabilityProbeReachable, CheckedAt: old}, + }, old) _ = (availabilityPollProvider{}).SupplementalRecords(monitor, "") for _, alert := range alertManager.GetActiveAlerts() { diff --git a/internal/monitoring/availability_udp_test.go b/internal/monitoring/availability_udp_test.go index e0b932712..b27bb31d3 100644 --- a/internal/monitoring/availability_udp_test.go +++ b/internal/monitoring/availability_udp_test.go @@ -86,7 +86,7 @@ func TestAvailabilityUDPIndeterminateDoesNotClaimReachabilityOrAccumulateFailure }} checkedAt := time.Now().UTC() - monitor.setAvailabilityStatus(target, checkedAt, 250*time.Millisecond, AvailabilityProbeIndeterminate, nil, "") + monitor.setAvailabilityStatus(target, checkedAt, 250*time.Millisecond, AvailabilityProbeIndeterminate, nil, "", time.Time{}) status := monitor.AvailabilityStatusSnapshot()[target.ID] if status.Available { t.Fatal("indeterminate UDP result must not be reported as reachable")