Rank a confirmed availability outage first in the summary check (#2664)

A machine with several availability checks shows one summary check, and
its health reads only that summary. The summary ranked every observed
failure equally and broke ties by target id. So a check that had failed
once, below its threshold, could hide a later-sorting check with a
confirmed outage. The outage check's own row carried
`availability_unreachable` while the machine it monitors read healthy.

- `availabilityCheckPriority` now ranks a confirmed outage first, then
any other observed failure, then unchecked/ambiguous/unresolved, then
passing. Ties still go to the first target id.
- `availabilityOutageConfirmed` is the one confirmation rule, shared by
the ranking and `EvaluateResourceHealth`. It mirrors the poller's
incident gate: observed failures at the threshold, an unavailable
aggregate when one is reported, and no indeterminate outcome. A probe
agent that stops reporting keeps its old failure count but confirms
nothing, on its own row or on the target.
- Tests in `availability_link_test.go` go through the registry
projection path: a confirmed outage behind an earlier unconfirmed
failure (and its recovery), and a silent probe agent's stale failure
count. Both fail without the change. `health_test.go`'s legacy fixture
gains the `LastChecked` every producer sets.
- The `unified-resources.md` contract describes the summary ranking and
the shared rule.

Drawers already list every check from `availabilityChecks`, and
availability incidents live on each check's own row, so alert synthesis
is unchanged.

docs/release-control/v6/internal/subsystems/unified-resources.md

@@ -3431,6 +3431,20 @@ It owns probe status, incidents, history, evidence, and the outgoing `checks`
 relationship. The matched resource carries every correlated check in the canonical
 `availabilityChecks` facet, keyed by saved target id, while `availability`
 remains an additive singular compatibility summary selected from that set.
+The summary is the worst check: a confirmed outage first, then any other
+observed failure, then an unchecked, ambiguous or unresolved check, then a
+passing one, ties going to the first saved target id. A confirmed outage is an
+enabled check whose observed failures reach its failure threshold, with an
+unavailable aggregate when one is reported and an outcome that is not
+indeterminate. That is the gate the availability poller applies before it
+raises `availability_unreachable`, so a probe agent that stops reporting keeps
+its old failure count but confirms nothing. `availabilityOutageConfirmed` in
+`internal/unifiedresources/availability.go` holds that rule for both the
+ranking and `EvaluateResourceHealth`, which reads the summary. A confirmed
+outage on any attached check therefore makes the resource's health critical
+even when a check that sorts earlier has failed below its threshold, with the
+`availability_failed` reason unless a critical alert explains the verdict
+first.
 Adding a second explicit or unambiguously correlated check must retain both
 source-owned endpoint rows, project both facets onto the same resource, and
 emit one `checks` relationship per target from the check to that resource.

internal/unifiedresources/availability.go

@@ -121,6 +121,10 @@ func availabilityCheckKey(check AvailabilityData) string {
 	return fmt.Sprintf("endpoint:%s:%s:%d:%s", protocol, address, check.Port, path)
 }
 
+// primaryAvailabilityCheck selects the singular compatibility summary: the
+// worst check, ties going to the first by key. Health and row presentation
+// read only the summary, so a confirmed outage on any check must outrank a
+// failure that has not yet reached its threshold.
 func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData {
 	if len(checks) == 0 {
 		return nil
@@ -135,13 +139,39 @@ func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData {
 }
 
 func availabilityCheckPriority(check AvailabilityData) int {
-	if check.LastChecked != nil && !check.Available {
+	if availabilityOutageConfirmed(check) {
 		return 0
 	}
+	if check.LastChecked != nil && !check.Available {
+		return 1
+	}
 	if check.LastChecked == nil ||
 		check.CorrelationState == AvailabilityCorrelationAmbiguous ||
 		check.CorrelationState == AvailabilityCorrelationUnresolved {
-		return 1
+		return 2
 	}
-	return 2
+	return 3
+}
+
+// availabilityOutageConfirmed reports whether an enabled check has failed at
+// least its failure threshold in a row, by the same gate the availability
+// poller uses before raising availability_unreachable: an observed failure
+// whose aggregate, when present, is unavailable. A probe agent that stops
+// reporting keeps its failure count but reads indeterminate with an unknown
+// aggregate, and confirms nothing.
+func availabilityOutageConfirmed(check AvailabilityData) bool {
+	if !check.Enabled || check.Available || check.LastChecked == nil {
+		return false
+	}
+	if state := strings.TrimSpace(check.AggregateState); state != "" && !strings.EqualFold(state, "unavailable") {
+		return false
+	}
+	if strings.EqualFold(strings.TrimSpace(check.ProbeOutcome), "indeterminate") {
+		return false
+	}
+	threshold := check.FailureThreshold
+	if threshold <= 0 {
+		threshold = 1
+	}
+	return check.ConsecutiveFailures >= threshold
 }

internal/unifiedresources/availability_link_test.go

@@ -403,6 +403,133 @@ func TestAvailabilityKeepsEveryConfiguredCheckWithMultipleServicesOnOneHost(t *t
 	}
 }
 
+func TestAvailabilitySummaryPrefersConfirmedOutageOverEarlierUnconfirmedFailure(t *testing.T) {
+	rr := NewRegistry(nil)
+	hostID := ingestAgentFixture(t, rr, "host-1", "machine-1")
+	now := time.Now().UTC()
+	check := func(targetID, address string, facet AvailabilityData) IngestRecord {
+		checkedAt := now
+		facet.LinkedResourceID = hostID
+		facet.Address = address
+		facet.Protocol = "https"
+		facet.Enabled = true
+		facet.LastChecked = &checkedAt
+		facet.FailureThreshold = 3
+		return availabilityProbeRecord(targetID, address, &facet)
+	}
+
+	// probe-a sorts first by target id and has failed once, below its
+	// threshold. probe-b is past its threshold and owns the outage incident.
+	unconfirmed := check("probe-a", "203.0.113.10", AvailabilityData{
+		AggregateState:         "unavailable",
+		ConsecutiveFailures:    1,
+		ApplicationFailureCode: "status_mismatch",
+	})
+	unconfirmed.Resource.Status = StatusWarning
+	confirmed := check("probe-b", "203.0.113.11", AvailabilityData{
+		AggregateState:      "unavailable",
+		ConsecutiveFailures: 3,
+	})
+	confirmed.Resource.Status = StatusOffline
+	confirmed.Resource.Incidents = []ResourceIncident{{
+		Provider: string(SourceAvailability),
+		NativeID: "probe-b",
+		Code:     "availability_unreachable",
+	}}
+	rr.IngestRecords(SourceAvailability, []IngestRecord{unconfirmed, confirmed})
+
+	host, ok := rr.Get(hostID)
+	if !ok || host == nil {
+		t.Fatalf("host %q missing after ingest", hostID)
+	}
+	if len(AvailabilityChecksForResource(*host)) != 2 {
+		t.Fatalf("availability checks = %+v, want both attached checks", host.AvailabilityChecks)
+	}
+	if host.Availability == nil || host.Availability.TargetID != "probe-b" {
+		t.Fatalf("availability summary = %+v, want the confirmed outage on probe-b", host.Availability)
+	}
+	health := EvaluateResourceHealth(*host, nil, now)
+	if health.Verdict != HealthCritical || len(health.Reasons) == 0 ||
+		health.Reasons[0].Code != "availability_failed" || health.Reasons[0].Detail != "" {
+		t.Fatalf("host health = %+v, want availability_failed from probe-b", health)
+	}
+
+	// Once probe-b recovers, the unconfirmed failure is the worst check left:
+	// it is the summary again, and it does not make the host an outage.
+	rr.IngestRecords(SourceAvailability, []IngestRecord{
+		check("probe-b", "203.0.113.11", AvailabilityData{
+			AggregateState: "healthy",
+			Available:      true,
+		}),
+	})
+	host, _ = rr.Get(hostID)
+	if host.Availability == nil || host.Availability.TargetID != "probe-a" {
+		t.Fatalf("availability summary = %+v, want the remaining failure on probe-a", host.Availability)
+	}
+	health = EvaluateResourceHealth(*host, nil, now)
+	for _, reason := range health.Reasons {
+		if reason.Code == "availability_failed" {
+			t.Fatalf("host health = %+v, want no outage below the failure threshold", health)
+		}
+	}
+}
+
+func TestAvailabilitySummaryDoesNotConfirmOutageFromSilentProbeAgent(t *testing.T) {
+	rr := NewRegistry(nil)
+	hostID := ingestAgentFixture(t, rr, "host-1", "machine-1")
+	now := time.Now().UTC()
+	checkedAt := now
+	unconfirmed := availabilityProbeRecord("probe-a", "203.0.113.10", &AvailabilityData{
+		LinkedResourceID:    hostID,
+		Address:             "203.0.113.10",
+		Protocol:            "https",
+		Enabled:             true,
+		AggregateState:      "unavailable",
+		ProbeOutcome:        "unreachable",
+		LastChecked:         &checkedAt,
+		ConsecutiveFailures: 1,
+		FailureThreshold:    3,
+	})
+	unconfirmed.Resource.Status = StatusWarning
+
+	// The poller's read of a single-location probe whose agent stopped
+	// reporting: the old failure count survives, but the outcome is
+	// indeterminate, the aggregate unknown, and no incident is raised.
+	lastReport := now.Add(-time.Hour)
+	silent := availabilityProbeRecord("probe-b", "203.0.113.11", &AvailabilityData{
+		LinkedResourceID:    hostID,
+		Address:             "203.0.113.11",
+		Protocol:            "https",
+		ProbeAgentID:        "agent-probe-1",
+		Enabled:             true,
+		AggregateState:      "unknown",
+		ProbeOutcome:        "indeterminate",
+		ExpectedLocations:   1,
+		LastChecked:         &lastReport,
+		ConsecutiveFailures: 5,
+		FailureThreshold:    3,
+		LastError:           "no recent report from probe agent",
+	})
+	silent.Resource.Status = StatusWarning
+	rr.IngestRecords(SourceAvailability, []IngestRecord{unconfirmed, silent})
+
+	host, ok := rr.Get(hostID)
+	if !ok || host == nil {
+		t.Fatalf("host %q missing after ingest", hostID)
+	}
+	if host.Availability == nil || host.Availability.TargetID != "probe-a" {
+		t.Fatalf("availability summary = %+v, want probe-a: a silent probe confirms no outage", host.Availability)
+	}
+	for _, resource := range []Resource{*host, availabilityEndpointByTarget(t, rr, "probe-b")} {
+		health := EvaluateResourceHealth(resource, nil, now)
+		for _, reason := range health.Reasons {
+			if reason.Code == "availability_failed" {
+				t.Fatalf("%s health = %+v, want no outage from a silent probe agent", resource.ID, health)
+			}
+		}
+	}
+}
+
 func TestAvailabilityEditReplacesEndpointAndMovesProjection(t *testing.T) {
 	rr := NewRegistry(nil)
 	hostA := ingestAgentFixture(t, rr, "host-a", "machine-a")

internal/unifiedresources/health.go

@@ -187,15 +187,10 @@ func healthAlertSeverity(level string) int {
 	}
 }
 
+// availabilityFailureConfirmed reads the compatibility summary, which ranks a
+// confirmed outage on any attached check first.
 func availabilityFailureConfirmed(resource Resource) bool {
-	if resource.Availability == nil || !resource.Availability.Enabled || resource.Availability.Available {
-		return false
-	}
-	threshold := resource.Availability.FailureThreshold
-	if threshold <= 0 {
-		threshold = 1
-	}
-	return resource.Availability.ConsecutiveFailures >= threshold
+	return resource.Availability != nil && availabilityOutageConfirmed(*resource.Availability)
 }
 
 func availabilityFailureDetail(resource Resource) string {

internal/unifiedresources/health_test.go

@@ -31,7 +31,7 @@ func TestEvaluateResourceHealthPrecedenceAndVerdicts(t *testing.T) {
 		},
 		{
 			name:     "confirmed availability failure is critical",
-			resource: Resource{ID: "check-1", Type: ResourceTypeNetworkEndpoint, Status: StatusOnline, Availability: &AvailabilityData{Enabled: true, Available: false, ConsecutiveFailures: 3, FailureThreshold: 3}},
+			resource: Resource{ID: "check-1", Type: ResourceTypeNetworkEndpoint, Status: StatusOnline, Availability: &AvailabilityData{Enabled: true, Available: false, LastChecked: &now, ConsecutiveFailures: 3, FailureThreshold: 3}},
 			want:     HealthCritical, code: "availability_failed",
 		},
 		{