Files
pulse/internal/unifiedresources/availability.go
T
rcourtman 900d620abd Rank a confirmed availability outage first in the summary check (#2664)
A machine with several availability checks shows one summary check, and
its health reads only that summary. The summary ranked every observed
failure equally and broke ties by target id. So a check that had failed
once, below its threshold, could hide a later-sorting check with a
confirmed outage. The outage check's own row carried
`availability_unreachable` while the machine it monitors read healthy.

- `availabilityCheckPriority` now ranks a confirmed outage first, then
any other observed failure, then unchecked/ambiguous/unresolved, then
passing. Ties still go to the first target id.
- `availabilityOutageConfirmed` is the one confirmation rule, shared by
the ranking and `EvaluateResourceHealth`. It mirrors the poller's
incident gate: observed failures at the threshold, an unavailable
aggregate when one is reported, and no indeterminate outcome. A probe
agent that stops reporting keeps its old failure count but confirms
nothing, on its own row or on the target.
- Tests in `availability_link_test.go` go through the registry
projection path: a confirmed outage behind an earlier unconfirmed
failure (and its recovery), and a silent probe agent's stale failure
count. Both fail without the change. `health_test.go`'s legacy fixture
gains the `LastChecked` every producer sets.
- The `unified-resources.md` contract describes the summary ranking and
the shared rule.

Drawers already list every check from `availabilityChecks`, and
availability incidents live on each check's own row, so alert synthesis
is unchanged.
2026-10-07 15:23:46 +01:00

178 lines
5.6 KiB
Go

package unifiedresources
import (
"fmt"
"sort"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
)
// AvailabilityChecksForResource returns the complete canonical availability
// facet set. The singular Availability field is retained as an additive
// compatibility summary and is folded into the result when older payloads do
// not yet carry AvailabilityChecks.
func AvailabilityChecksForResource(resource Resource) []AvailabilityData {
return mergeAvailabilityChecks(nil, nil, resource.AvailabilityChecks, resource.Availability)
}
// AvailabilityCheckByTargetID returns the observation facet associated with a
// provider target. Alert and evidence consumers use this instead of assuming
// the compatibility summary is the incident's originating check.
func AvailabilityCheckByTargetID(resource Resource, targetID string) *AvailabilityData {
targetID = strings.TrimSpace(targetID)
for _, check := range AvailabilityChecksForResource(resource) {
if strings.TrimSpace(check.TargetID) == targetID {
cloned := cloneAvailabilityData(&check)
return cloned
}
}
return nil
}
// availabilityChecksProveOnline reports whether any availability check
// projected onto a monitored resource proves, at now, that the resource
// answers. Only a passing check with current evidence does. Each check is
// judged by its own evidence window (two poll intervals for a local check,
// the report window for a remote probe), so one check's recent run cannot
// vouch for another check's old pass, and a local check that missed its
// cadence keeps Available but proves nothing. A failing check proves only
// that one port or service does not answer, so it never decides the
// resource's status at all: not online, offline or warning, current or quiet.
func availabilityChecksProveOnline(checks []AvailabilityData, now time.Time) bool {
for _, check := range checks {
if check.Enabled && check.Available && check.Evidence != nil &&
check.Evidence.FreshnessAt(now) == operationaltrust.EvidenceFresh {
return true
}
}
return false
}
func normalizeResourceAvailability(resource *Resource) {
if resource == nil {
return
}
resource.AvailabilityChecks = mergeAvailabilityChecks(
nil,
nil,
resource.AvailabilityChecks,
resource.Availability,
)
resource.Availability = primaryAvailabilityCheck(resource.AvailabilityChecks)
}
func mergeAvailabilityChecks(
existing []AvailabilityData,
existingPrimary *AvailabilityData,
incoming []AvailabilityData,
incomingPrimary *AvailabilityData,
) []AvailabilityData {
byKey := make(map[string]AvailabilityData, len(existing)+len(incoming)+2)
add := func(check AvailabilityData) {
key := availabilityCheckKey(check)
if key == "" {
return
}
cloned := cloneAvailabilityData(&check)
if cloned != nil {
byKey[key] = *cloned
}
}
for _, check := range existing {
add(check)
}
if existingPrimary != nil {
add(*existingPrimary)
}
for _, check := range incoming {
add(check)
}
if incomingPrimary != nil {
add(*incomingPrimary)
}
if len(byKey) == 0 {
return nil
}
keys := make([]string, 0, len(byKey))
for key := range byKey {
keys = append(keys, key)
}
sort.Strings(keys)
out := make([]AvailabilityData, 0, len(keys))
for _, key := range keys {
out = append(out, byKey[key])
}
return out
}
func availabilityCheckKey(check AvailabilityData) string {
if targetID := strings.TrimSpace(check.TargetID); targetID != "" {
return "target:" + targetID
}
address := strings.ToLower(strings.TrimSpace(check.Address))
protocol := strings.ToLower(strings.TrimSpace(check.Protocol))
path := strings.TrimSpace(check.Path)
if address == "" && protocol == "" && check.Port == 0 && path == "" {
return ""
}
return fmt.Sprintf("endpoint:%s:%s:%d:%s", protocol, address, check.Port, path)
}
// primaryAvailabilityCheck selects the singular compatibility summary: the
// worst check, ties going to the first by key. Health and row presentation
// read only the summary, so a confirmed outage on any check must outrank a
// failure that has not yet reached its threshold.
func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData {
if len(checks) == 0 {
return nil
}
best := 0
for index := 1; index < len(checks); index++ {
if availabilityCheckPriority(checks[index]) < availabilityCheckPriority(checks[best]) {
best = index
}
}
return cloneAvailabilityData(&checks[best])
}
func availabilityCheckPriority(check AvailabilityData) int {
if availabilityOutageConfirmed(check) {
return 0
}
if check.LastChecked != nil && !check.Available {
return 1
}
if check.LastChecked == nil ||
check.CorrelationState == AvailabilityCorrelationAmbiguous ||
check.CorrelationState == AvailabilityCorrelationUnresolved {
return 2
}
return 3
}
// availabilityOutageConfirmed reports whether an enabled check has failed at
// least its failure threshold in a row, by the same gate the availability
// poller uses before raising availability_unreachable: an observed failure
// whose aggregate, when present, is unavailable. A probe agent that stops
// reporting keeps its failure count but reads indeterminate with an unknown
// aggregate, and confirms nothing.
func availabilityOutageConfirmed(check AvailabilityData) bool {
if !check.Enabled || check.Available || check.LastChecked == nil {
return false
}
if state := strings.TrimSpace(check.AggregateState); state != "" && !strings.EqualFold(state, "unavailable") {
return false
}
if strings.EqualFold(strings.TrimSpace(check.ProbeOutcome), "indeterminate") {
return false
}
threshold := check.FailureThreshold
if threshold <= 0 {
threshold = 1
}
return check.ConsecutiveFailures >= threshold
}