diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 53e49325b6..5a4ebb001d 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -3443,6 +3443,20 @@ It owns probe status, incidents, history, evidence, and the outgoing `checks` relationship. The matched resource carries every correlated check in the canonical `availabilityChecks` facet, keyed by saved target id, while `availability` remains an additive singular compatibility summary selected from that set. +The summary is the worst check: a confirmed outage first, then any other +observed failure, then an unchecked, ambiguous or unresolved check, then a +passing one, ties going to the first saved target id. A confirmed outage is an +enabled check whose observed failures reach its failure threshold, with an +unavailable aggregate when one is reported and an outcome that is not +indeterminate. That is the gate the availability poller applies before it +raises `availability_unreachable`, so a probe agent that stops reporting keeps +its old failure count but confirms nothing. `availabilityOutageConfirmed` in +`internal/unifiedresources/availability.go` holds that rule for both the +ranking and `EvaluateResourceHealth`, which reads the summary. A confirmed +outage on any attached check therefore makes the resource's health critical +even when a check that sorts earlier has failed below its threshold, with the +`availability_failed` reason unless a critical alert explains the verdict +first. Adding a second explicit or unambiguously correlated check must retain both source-owned endpoint rows, project both facets onto the same resource, and emit one `checks` relationship per target from the check to that resource. diff --git a/internal/unifiedresources/availability.go b/internal/unifiedresources/availability.go index 35620d04da..87c5d9ac21 100644 --- a/internal/unifiedresources/availability.go +++ b/internal/unifiedresources/availability.go @@ -99,6 +99,10 @@ func availabilityCheckKey(check AvailabilityData) string { return fmt.Sprintf("endpoint:%s:%s:%d:%s", protocol, address, check.Port, path) } +// primaryAvailabilityCheck selects the singular compatibility summary: the +// worst check, ties going to the first by key. Health and row presentation +// read only the summary, so a confirmed outage on any check must outrank a +// failure that has not yet reached its threshold. func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData { if len(checks) == 0 { return nil @@ -113,13 +117,39 @@ func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData { } func availabilityCheckPriority(check AvailabilityData) int { - if check.LastChecked != nil && !check.Available { + if availabilityOutageConfirmed(check) { return 0 } + if check.LastChecked != nil && !check.Available { + return 1 + } if check.LastChecked == nil || check.CorrelationState == AvailabilityCorrelationAmbiguous || check.CorrelationState == AvailabilityCorrelationUnresolved { - return 1 + return 2 } - return 2 + return 3 +} + +// availabilityOutageConfirmed reports whether an enabled check has failed at +// least its failure threshold in a row, by the same gate the availability +// poller uses before raising availability_unreachable: an observed failure +// whose aggregate, when present, is unavailable. A probe agent that stops +// reporting keeps its failure count but reads indeterminate with an unknown +// aggregate, and confirms nothing. +func availabilityOutageConfirmed(check AvailabilityData) bool { + if !check.Enabled || check.Available || check.LastChecked == nil { + return false + } + if state := strings.TrimSpace(check.AggregateState); state != "" && !strings.EqualFold(state, "unavailable") { + return false + } + if strings.EqualFold(strings.TrimSpace(check.ProbeOutcome), "indeterminate") { + return false + } + threshold := check.FailureThreshold + if threshold <= 0 { + threshold = 1 + } + return check.ConsecutiveFailures >= threshold } diff --git a/internal/unifiedresources/availability_link_test.go b/internal/unifiedresources/availability_link_test.go index 1daaa01432..fcb21d0289 100644 --- a/internal/unifiedresources/availability_link_test.go +++ b/internal/unifiedresources/availability_link_test.go @@ -403,6 +403,133 @@ func TestAvailabilityKeepsEveryConfiguredCheckWithMultipleServicesOnOneHost(t *t } } +func TestAvailabilitySummaryPrefersConfirmedOutageOverEarlierUnconfirmedFailure(t *testing.T) { + rr := NewRegistry(nil) + hostID := ingestAgentFixture(t, rr, "host-1", "machine-1") + now := time.Now().UTC() + check := func(targetID, address string, facet AvailabilityData) IngestRecord { + checkedAt := now + facet.LinkedResourceID = hostID + facet.Address = address + facet.Protocol = "https" + facet.Enabled = true + facet.LastChecked = &checkedAt + facet.FailureThreshold = 3 + return availabilityProbeRecord(targetID, address, &facet) + } + + // probe-a sorts first by target id and has failed once, below its + // threshold. probe-b is past its threshold and owns the outage incident. + unconfirmed := check("probe-a", "203.0.113.10", AvailabilityData{ + AggregateState: "unavailable", + ConsecutiveFailures: 1, + ApplicationFailureCode: "status_mismatch", + }) + unconfirmed.Resource.Status = StatusWarning + confirmed := check("probe-b", "203.0.113.11", AvailabilityData{ + AggregateState: "unavailable", + ConsecutiveFailures: 3, + }) + confirmed.Resource.Status = StatusOffline + confirmed.Resource.Incidents = []ResourceIncident{{ + Provider: string(SourceAvailability), + NativeID: "probe-b", + Code: "availability_unreachable", + }} + rr.IngestRecords(SourceAvailability, []IngestRecord{unconfirmed, confirmed}) + + host, ok := rr.Get(hostID) + if !ok || host == nil { + t.Fatalf("host %q missing after ingest", hostID) + } + if len(AvailabilityChecksForResource(*host)) != 2 { + t.Fatalf("availability checks = %+v, want both attached checks", host.AvailabilityChecks) + } + if host.Availability == nil || host.Availability.TargetID != "probe-b" { + t.Fatalf("availability summary = %+v, want the confirmed outage on probe-b", host.Availability) + } + health := EvaluateResourceHealth(*host, nil, now) + if health.Verdict != HealthCritical || len(health.Reasons) == 0 || + health.Reasons[0].Code != "availability_failed" || health.Reasons[0].Detail != "" { + t.Fatalf("host health = %+v, want availability_failed from probe-b", health) + } + + // Once probe-b recovers, the unconfirmed failure is the worst check left: + // it is the summary again, and it does not make the host an outage. + rr.IngestRecords(SourceAvailability, []IngestRecord{ + check("probe-b", "203.0.113.11", AvailabilityData{ + AggregateState: "healthy", + Available: true, + }), + }) + host, _ = rr.Get(hostID) + if host.Availability == nil || host.Availability.TargetID != "probe-a" { + t.Fatalf("availability summary = %+v, want the remaining failure on probe-a", host.Availability) + } + health = EvaluateResourceHealth(*host, nil, now) + for _, reason := range health.Reasons { + if reason.Code == "availability_failed" { + t.Fatalf("host health = %+v, want no outage below the failure threshold", health) + } + } +} + +func TestAvailabilitySummaryDoesNotConfirmOutageFromSilentProbeAgent(t *testing.T) { + rr := NewRegistry(nil) + hostID := ingestAgentFixture(t, rr, "host-1", "machine-1") + now := time.Now().UTC() + checkedAt := now + unconfirmed := availabilityProbeRecord("probe-a", "203.0.113.10", &AvailabilityData{ + LinkedResourceID: hostID, + Address: "203.0.113.10", + Protocol: "https", + Enabled: true, + AggregateState: "unavailable", + ProbeOutcome: "unreachable", + LastChecked: &checkedAt, + ConsecutiveFailures: 1, + FailureThreshold: 3, + }) + unconfirmed.Resource.Status = StatusWarning + + // The poller's read of a single-location probe whose agent stopped + // reporting: the old failure count survives, but the outcome is + // indeterminate, the aggregate unknown, and no incident is raised. + lastReport := now.Add(-time.Hour) + silent := availabilityProbeRecord("probe-b", "203.0.113.11", &AvailabilityData{ + LinkedResourceID: hostID, + Address: "203.0.113.11", + Protocol: "https", + ProbeAgentID: "agent-probe-1", + Enabled: true, + AggregateState: "unknown", + ProbeOutcome: "indeterminate", + ExpectedLocations: 1, + LastChecked: &lastReport, + ConsecutiveFailures: 5, + FailureThreshold: 3, + LastError: "no recent report from probe agent", + }) + silent.Resource.Status = StatusWarning + rr.IngestRecords(SourceAvailability, []IngestRecord{unconfirmed, silent}) + + host, ok := rr.Get(hostID) + if !ok || host == nil { + t.Fatalf("host %q missing after ingest", hostID) + } + if host.Availability == nil || host.Availability.TargetID != "probe-a" { + t.Fatalf("availability summary = %+v, want probe-a: a silent probe confirms no outage", host.Availability) + } + for _, resource := range []Resource{*host, availabilityEndpointByTarget(t, rr, "probe-b")} { + health := EvaluateResourceHealth(resource, nil, now) + for _, reason := range health.Reasons { + if reason.Code == "availability_failed" { + t.Fatalf("%s health = %+v, want no outage from a silent probe agent", resource.ID, health) + } + } + } +} + func TestAvailabilityEditReplacesEndpointAndMovesProjection(t *testing.T) { rr := NewRegistry(nil) hostA := ingestAgentFixture(t, rr, "host-a", "machine-a") diff --git a/internal/unifiedresources/health.go b/internal/unifiedresources/health.go index fabf9c7bd0..ef74dc69ea 100644 --- a/internal/unifiedresources/health.go +++ b/internal/unifiedresources/health.go @@ -187,15 +187,10 @@ func healthAlertSeverity(level string) int { } } +// availabilityFailureConfirmed reads the compatibility summary, which ranks a +// confirmed outage on any attached check first. func availabilityFailureConfirmed(resource Resource) bool { - if resource.Availability == nil || !resource.Availability.Enabled || resource.Availability.Available { - return false - } - threshold := resource.Availability.FailureThreshold - if threshold <= 0 { - threshold = 1 - } - return resource.Availability.ConsecutiveFailures >= threshold + return resource.Availability != nil && availabilityOutageConfirmed(*resource.Availability) } func availabilityFailureDetail(resource Resource) string { diff --git a/internal/unifiedresources/health_test.go b/internal/unifiedresources/health_test.go index c51de693e4..696733d0cb 100644 --- a/internal/unifiedresources/health_test.go +++ b/internal/unifiedresources/health_test.go @@ -31,7 +31,7 @@ func TestEvaluateResourceHealthPrecedenceAndVerdicts(t *testing.T) { }, { name: "confirmed availability failure is critical", - resource: Resource{ID: "check-1", Type: ResourceTypeNetworkEndpoint, Status: StatusOnline, Availability: &AvailabilityData{Enabled: true, Available: false, ConsecutiveFailures: 3, FailureThreshold: 3}}, + resource: Resource{ID: "check-1", Type: ResourceTypeNetworkEndpoint, Status: StatusOnline, Availability: &AvailabilityData{Enabled: true, Available: false, LastChecked: &now, ConsecutiveFailures: 3, FailureThreshold: 3}}, want: HealthCritical, code: "availability_failed", }, {