From ade799307a54de6ea12325251cc2f99347d33385 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:27:20 +0100 Subject: [PATCH] Rank a confirmed availability outage first in the summary check A resource with several attached availability checks exposes one compatibility summary, Resource.Availability, and health reads only that. The summary ranked every observed failure equally and broke ties by target id, so a check that had failed once below its threshold could hide a later-sorting check with a confirmed outage: the outage check's own row carried availability_unreachable while the machine it monitors read healthy. Rank a confirmed outage above any other failure and have health use the same per-check rule, so target health and row presentation follow a confirmed outage on any attached check. The rule mirrors the poller's incident gate: observed failures at the threshold, an unavailable aggregate when one is reported, and no indeterminate outcome. A probe agent that stops reporting keeps its old failure count but reads indeterminate with an unknown aggregate, so it confirms nothing, on its own row or on the target. Drawers already list every check from availabilityChecks; the summary keeps its worst-check meaning. --- .../internal/subsystems/unified-resources.md | 14 ++ internal/unifiedresources/availability.go | 36 ++++- .../availability_link_test.go | 127 ++++++++++++++++++ internal/unifiedresources/health.go | 11 +- internal/unifiedresources/health_test.go | 2 +- 5 files changed, 178 insertions(+), 12 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 53e49325b6..5a4ebb001d 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -3443,6 +3443,20 @@ It owns probe status, incidents, history, evidence, and the outgoing `checks` relationship. The matched resource carries every correlated check in the canonical `availabilityChecks` facet, keyed by saved target id, while `availability` remains an additive singular compatibility summary selected from that set. +The summary is the worst check: a confirmed outage first, then any other +observed failure, then an unchecked, ambiguous or unresolved check, then a +passing one, ties going to the first saved target id. A confirmed outage is an +enabled check whose observed failures reach its failure threshold, with an +unavailable aggregate when one is reported and an outcome that is not +indeterminate. That is the gate the availability poller applies before it +raises `availability_unreachable`, so a probe agent that stops reporting keeps +its old failure count but confirms nothing. `availabilityOutageConfirmed` in +`internal/unifiedresources/availability.go` holds that rule for both the +ranking and `EvaluateResourceHealth`, which reads the summary. A confirmed +outage on any attached check therefore makes the resource's health critical +even when a check that sorts earlier has failed below its threshold, with the +`availability_failed` reason unless a critical alert explains the verdict +first. Adding a second explicit or unambiguously correlated check must retain both source-owned endpoint rows, project both facets onto the same resource, and emit one `checks` relationship per target from the check to that resource. diff --git a/internal/unifiedresources/availability.go b/internal/unifiedresources/availability.go index 35620d04da..87c5d9ac21 100644 --- a/internal/unifiedresources/availability.go +++ b/internal/unifiedresources/availability.go @@ -99,6 +99,10 @@ func availabilityCheckKey(check AvailabilityData) string { return fmt.Sprintf("endpoint:%s:%s:%d:%s", protocol, address, check.Port, path) } +// primaryAvailabilityCheck selects the singular compatibility summary: the +// worst check, ties going to the first by key. Health and row presentation +// read only the summary, so a confirmed outage on any check must outrank a +// failure that has not yet reached its threshold. func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData { if len(checks) == 0 { return nil @@ -113,13 +117,39 @@ func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData { } func availabilityCheckPriority(check AvailabilityData) int { - if check.LastChecked != nil && !check.Available { + if availabilityOutageConfirmed(check) { return 0 } + if check.LastChecked != nil && !check.Available { + return 1 + } if check.LastChecked == nil || check.CorrelationState == AvailabilityCorrelationAmbiguous || check.CorrelationState == AvailabilityCorrelationUnresolved { - return 1 + return 2 } - return 2 + return 3 +} + +// availabilityOutageConfirmed reports whether an enabled check has failed at +// least its failure threshold in a row, by the same gate the availability +// poller uses before raising availability_unreachable: an observed failure +// whose aggregate, when present, is unavailable. A probe agent that stops +// reporting keeps its failure count but reads indeterminate with an unknown +// aggregate, and confirms nothing. +func availabilityOutageConfirmed(check AvailabilityData) bool { + if !check.Enabled || check.Available || check.LastChecked == nil { + return false + } + if state := strings.TrimSpace(check.AggregateState); state != "" && !strings.EqualFold(state, "unavailable") { + return false + } + if strings.EqualFold(strings.TrimSpace(check.ProbeOutcome), "indeterminate") { + return false + } + threshold := check.FailureThreshold + if threshold <= 0 { + threshold = 1 + } + return check.ConsecutiveFailures >= threshold } diff --git a/internal/unifiedresources/availability_link_test.go b/internal/unifiedresources/availability_link_test.go index 1daaa01432..fcb21d0289 100644 --- a/internal/unifiedresources/availability_link_test.go +++ b/internal/unifiedresources/availability_link_test.go @@ -403,6 +403,133 @@ func TestAvailabilityKeepsEveryConfiguredCheckWithMultipleServicesOnOneHost(t *t } } +func TestAvailabilitySummaryPrefersConfirmedOutageOverEarlierUnconfirmedFailure(t *testing.T) { + rr := NewRegistry(nil) + hostID := ingestAgentFixture(t, rr, "host-1", "machine-1") + now := time.Now().UTC() + check := func(targetID, address string, facet AvailabilityData) IngestRecord { + checkedAt := now + facet.LinkedResourceID = hostID + facet.Address = address + facet.Protocol = "https" + facet.Enabled = true + facet.LastChecked = &checkedAt + facet.FailureThreshold = 3 + return availabilityProbeRecord(targetID, address, &facet) + } + + // probe-a sorts first by target id and has failed once, below its + // threshold. probe-b is past its threshold and owns the outage incident. + unconfirmed := check("probe-a", "203.0.113.10", AvailabilityData{ + AggregateState: "unavailable", + ConsecutiveFailures: 1, + ApplicationFailureCode: "status_mismatch", + }) + unconfirmed.Resource.Status = StatusWarning + confirmed := check("probe-b", "203.0.113.11", AvailabilityData{ + AggregateState: "unavailable", + ConsecutiveFailures: 3, + }) + confirmed.Resource.Status = StatusOffline + confirmed.Resource.Incidents = []ResourceIncident{{ + Provider: string(SourceAvailability), + NativeID: "probe-b", + Code: "availability_unreachable", + }} + rr.IngestRecords(SourceAvailability, []IngestRecord{unconfirmed, confirmed}) + + host, ok := rr.Get(hostID) + if !ok || host == nil { + t.Fatalf("host %q missing after ingest", hostID) + } + if len(AvailabilityChecksForResource(*host)) != 2 { + t.Fatalf("availability checks = %+v, want both attached checks", host.AvailabilityChecks) + } + if host.Availability == nil || host.Availability.TargetID != "probe-b" { + t.Fatalf("availability summary = %+v, want the confirmed outage on probe-b", host.Availability) + } + health := EvaluateResourceHealth(*host, nil, now) + if health.Verdict != HealthCritical || len(health.Reasons) == 0 || + health.Reasons[0].Code != "availability_failed" || health.Reasons[0].Detail != "" { + t.Fatalf("host health = %+v, want availability_failed from probe-b", health) + } + + // Once probe-b recovers, the unconfirmed failure is the worst check left: + // it is the summary again, and it does not make the host an outage. + rr.IngestRecords(SourceAvailability, []IngestRecord{ + check("probe-b", "203.0.113.11", AvailabilityData{ + AggregateState: "healthy", + Available: true, + }), + }) + host, _ = rr.Get(hostID) + if host.Availability == nil || host.Availability.TargetID != "probe-a" { + t.Fatalf("availability summary = %+v, want the remaining failure on probe-a", host.Availability) + } + health = EvaluateResourceHealth(*host, nil, now) + for _, reason := range health.Reasons { + if reason.Code == "availability_failed" { + t.Fatalf("host health = %+v, want no outage below the failure threshold", health) + } + } +} + +func TestAvailabilitySummaryDoesNotConfirmOutageFromSilentProbeAgent(t *testing.T) { + rr := NewRegistry(nil) + hostID := ingestAgentFixture(t, rr, "host-1", "machine-1") + now := time.Now().UTC() + checkedAt := now + unconfirmed := availabilityProbeRecord("probe-a", "203.0.113.10", &AvailabilityData{ + LinkedResourceID: hostID, + Address: "203.0.113.10", + Protocol: "https", + Enabled: true, + AggregateState: "unavailable", + ProbeOutcome: "unreachable", + LastChecked: &checkedAt, + ConsecutiveFailures: 1, + FailureThreshold: 3, + }) + unconfirmed.Resource.Status = StatusWarning + + // The poller's read of a single-location probe whose agent stopped + // reporting: the old failure count survives, but the outcome is + // indeterminate, the aggregate unknown, and no incident is raised. + lastReport := now.Add(-time.Hour) + silent := availabilityProbeRecord("probe-b", "203.0.113.11", &AvailabilityData{ + LinkedResourceID: hostID, + Address: "203.0.113.11", + Protocol: "https", + ProbeAgentID: "agent-probe-1", + Enabled: true, + AggregateState: "unknown", + ProbeOutcome: "indeterminate", + ExpectedLocations: 1, + LastChecked: &lastReport, + ConsecutiveFailures: 5, + FailureThreshold: 3, + LastError: "no recent report from probe agent", + }) + silent.Resource.Status = StatusWarning + rr.IngestRecords(SourceAvailability, []IngestRecord{unconfirmed, silent}) + + host, ok := rr.Get(hostID) + if !ok || host == nil { + t.Fatalf("host %q missing after ingest", hostID) + } + if host.Availability == nil || host.Availability.TargetID != "probe-a" { + t.Fatalf("availability summary = %+v, want probe-a: a silent probe confirms no outage", host.Availability) + } + for _, resource := range []Resource{*host, availabilityEndpointByTarget(t, rr, "probe-b")} { + health := EvaluateResourceHealth(resource, nil, now) + for _, reason := range health.Reasons { + if reason.Code == "availability_failed" { + t.Fatalf("%s health = %+v, want no outage from a silent probe agent", resource.ID, health) + } + } + } +} + func TestAvailabilityEditReplacesEndpointAndMovesProjection(t *testing.T) { rr := NewRegistry(nil) hostA := ingestAgentFixture(t, rr, "host-a", "machine-a") diff --git a/internal/unifiedresources/health.go b/internal/unifiedresources/health.go index fabf9c7bd0..ef74dc69ea 100644 --- a/internal/unifiedresources/health.go +++ b/internal/unifiedresources/health.go @@ -187,15 +187,10 @@ func healthAlertSeverity(level string) int { } } +// availabilityFailureConfirmed reads the compatibility summary, which ranks a +// confirmed outage on any attached check first. func availabilityFailureConfirmed(resource Resource) bool { - if resource.Availability == nil || !resource.Availability.Enabled || resource.Availability.Available { - return false - } - threshold := resource.Availability.FailureThreshold - if threshold <= 0 { - threshold = 1 - } - return resource.Availability.ConsecutiveFailures >= threshold + return resource.Availability != nil && availabilityOutageConfirmed(*resource.Availability) } func availabilityFailureDetail(resource Resource) string { diff --git a/internal/unifiedresources/health_test.go b/internal/unifiedresources/health_test.go index c51de693e4..696733d0cb 100644 --- a/internal/unifiedresources/health_test.go +++ b/internal/unifiedresources/health_test.go @@ -31,7 +31,7 @@ func TestEvaluateResourceHealthPrecedenceAndVerdicts(t *testing.T) { }, { name: "confirmed availability failure is critical", - resource: Resource{ID: "check-1", Type: ResourceTypeNetworkEndpoint, Status: StatusOnline, Availability: &AvailabilityData{Enabled: true, Available: false, ConsecutiveFailures: 3, FailureThreshold: 3}}, + resource: Resource{ID: "check-1", Type: ResourceTypeNetworkEndpoint, Status: StatusOnline, Availability: &AvailabilityData{Enabled: true, Available: false, LastChecked: &now, ConsecutiveFailures: 3, FailureThreshold: 3}}, want: HealthCritical, code: "availability_failed", }, {