Let fresher evidence decide merged connection member state

When the Proxmox node inventory and the unified resource registry both
project the same machine as a system member, the merge paired the most
severe state from one plane with the newest LastSeen from the other. A
lapsed projection, such as an orphaned registry entry left behind by a
remove/re-enroll cycle, could therefore permanently render a member as
Stale beside a heartbeat seconds old, and the cluster rollup inherited
the contradiction onto the parent row.

The plane that has seen the machine more recently now decides liveness.
Severity still decides when neither side carries evidence or the
timestamps tie, so genuinely offline members keep their state.

Refs #1728
This commit is contained in:
Pulse Test
2026-08-30 00:02:15 +01:00
parent d607d5cf46
commit 54c3cdc061
2 changed files with 85 additions and 1 deletions
+32 -1
View File
@@ -345,7 +345,7 @@ func mergeConnectionSystemMembers(
if strings.TrimSpace(existing.Endpoint) == "" {
existing.Endpoint = candidate.Endpoint
}
existing.State = moreSevereConnectionSystemMemberState(existing.State, candidate.State)
existing.State = mergedConnectionSystemMemberState(existing, candidate)
if existing.LastSeen == nil ||
(candidate.LastSeen != nil && candidate.LastSeen.After(*existing.LastSeen)) {
existing.LastSeen = candidate.LastSeen
@@ -360,6 +360,37 @@ func mergeConnectionSystemMembers(
return existing
}
// mergedConnectionSystemMemberState resolves the state when the same machine
// is projected as a member by more than one observation plane (the Proxmox
// node inventory and the unified resource registry). The plane that has seen
// the machine more recently decides liveness; severity decides only when
// recency cannot (neither side carries evidence, or the timestamps tie).
// Merged LastSeen already keeps the newest timestamp, so this also guarantees
// the rendered badge and the rendered "Ns ago" come from the same plane — a
// lapsed projection, such as an orphaned registry entry left behind by a
// remove/re-enroll cycle, must not overrule a plane that heard from the
// machine seconds ago. Pairing worst state with newest timestamp rendered
// rows as "Stale · 5s ago".
func mergedConnectionSystemMemberState(
existing ConnectionSystemMember,
candidate ConnectionSystemMember,
) ConnectionState {
switch {
case existing.LastSeen == nil && candidate.LastSeen == nil:
return moreSevereConnectionSystemMemberState(existing.State, candidate.State)
case existing.LastSeen == nil:
return candidate.State
case candidate.LastSeen == nil:
return existing.State
case candidate.LastSeen.After(*existing.LastSeen):
return candidate.State
case existing.LastSeen.After(*candidate.LastSeen):
return existing.State
default:
return moreSevereConnectionSystemMemberState(existing.State, candidate.State)
}
}
func moreSevereConnectionSystemMemberState(
left ConnectionState,
right ConnectionState,
+53
View File
@@ -337,6 +337,59 @@ func TestMergeConnectionSystemMembersKeepsMostSevereState(t *testing.T) {
}
}
// A lapsed projection (orphaned unified-resource entry after a
// remove/re-enroll cycle) must not pair its severity with the live plane's
// fresh heartbeat: that rendered members as "Stale · 5s ago". The plane with
// the newer LastSeen decides the state; severity still wins when neither side
// carries evidence or the timestamps tie. Refs #1728.
func TestMergeConnectionSystemMembersFresherEvidenceDecidesState(t *testing.T) {
now := time.Now().UTC()
fresh := now.Add(-5 * time.Second)
lapsed := now.Add(-3 * 24 * time.Hour)
liveNode := ConnectionSystemMember{
ID: "node-minipc",
Name: "minipc",
State: ConnectionStateActive,
LastSeen: &fresh,
Primary: true,
}
orphanResource := ConnectionSystemMember{
ID: "resource-node-minipc",
Name: "minipc",
State: ConnectionStateStale,
LastSeen: &lapsed,
}
for name, merged := range map[string]ConnectionSystemMember{
"live-first": mergeConnectionSystemMembers(liveNode, orphanResource),
"orphan-first": mergeConnectionSystemMembers(orphanResource, liveNode),
} {
if merged.State != ConnectionStateActive {
t.Fatalf("%s: merged state = %q, want %q", name, merged.State, ConnectionStateActive)
}
if merged.LastSeen == nil || !merged.LastSeen.Equal(fresh) {
t.Fatalf("%s: merged lastSeen = %v, want %v", name, merged.LastSeen, fresh)
}
}
// An evidence-bearing severe plane still wins over a plane that has no
// evidence at all, and over an equally recent healthy plane.
unreachable := ConnectionSystemMember{
Name: "minipc",
State: ConnectionStateUnreachable,
LastSeen: &lapsed,
}
noEvidence := ConnectionSystemMember{Name: "minipc", State: ConnectionStateActive}
if merged := mergeConnectionSystemMembers(noEvidence, unreachable); merged.State != ConnectionStateUnreachable {
t.Fatalf("no-evidence merge state = %q, want %q", merged.State, ConnectionStateUnreachable)
}
tied := ConnectionSystemMember{Name: "minipc", State: ConnectionStateActive, LastSeen: &lapsed}
if merged := mergeConnectionSystemMembers(tied, unreachable); merged.State != ConnectionStateUnreachable {
t.Fatalf("tied-recency merge state = %q, want %q", merged.State, ConnectionStateUnreachable)
}
}
func TestConnectionSystemMemberUsesDisplayNameWithoutChangingMemberKey(t *testing.T) {
resource := unified.Resource{
ID: "resource-node",