mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-23 20:25:56 +00:00
Let fresher evidence decide merged connection member state
When the Proxmox node inventory and the unified resource registry both project the same machine as a system member, the merge paired the most severe state from one plane with the newest LastSeen from the other. A lapsed projection, such as an orphaned registry entry left behind by a remove/re-enroll cycle, could therefore permanently render a member as Stale beside a heartbeat seconds old, and the cluster rollup inherited the contradiction onto the parent row. The plane that has seen the machine more recently now decides liveness. Severity still decides when neither side carries evidence or the timestamps tie, so genuinely offline members keep their state. Refs #1728
This commit is contained in:
@@ -345,7 +345,7 @@ func mergeConnectionSystemMembers(
|
|||||||
if strings.TrimSpace(existing.Endpoint) == "" {
|
if strings.TrimSpace(existing.Endpoint) == "" {
|
||||||
existing.Endpoint = candidate.Endpoint
|
existing.Endpoint = candidate.Endpoint
|
||||||
}
|
}
|
||||||
existing.State = moreSevereConnectionSystemMemberState(existing.State, candidate.State)
|
existing.State = mergedConnectionSystemMemberState(existing, candidate)
|
||||||
if existing.LastSeen == nil ||
|
if existing.LastSeen == nil ||
|
||||||
(candidate.LastSeen != nil && candidate.LastSeen.After(*existing.LastSeen)) {
|
(candidate.LastSeen != nil && candidate.LastSeen.After(*existing.LastSeen)) {
|
||||||
existing.LastSeen = candidate.LastSeen
|
existing.LastSeen = candidate.LastSeen
|
||||||
@@ -360,6 +360,37 @@ func mergeConnectionSystemMembers(
|
|||||||
return existing
|
return existing
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mergedConnectionSystemMemberState resolves the state when the same machine
|
||||||
|
// is projected as a member by more than one observation plane (the Proxmox
|
||||||
|
// node inventory and the unified resource registry). The plane that has seen
|
||||||
|
// the machine more recently decides liveness; severity decides only when
|
||||||
|
// recency cannot (neither side carries evidence, or the timestamps tie).
|
||||||
|
// Merged LastSeen already keeps the newest timestamp, so this also guarantees
|
||||||
|
// the rendered badge and the rendered "Ns ago" come from the same plane — a
|
||||||
|
// lapsed projection, such as an orphaned registry entry left behind by a
|
||||||
|
// remove/re-enroll cycle, must not overrule a plane that heard from the
|
||||||
|
// machine seconds ago. Pairing worst state with newest timestamp rendered
|
||||||
|
// rows as "Stale · 5s ago".
|
||||||
|
func mergedConnectionSystemMemberState(
|
||||||
|
existing ConnectionSystemMember,
|
||||||
|
candidate ConnectionSystemMember,
|
||||||
|
) ConnectionState {
|
||||||
|
switch {
|
||||||
|
case existing.LastSeen == nil && candidate.LastSeen == nil:
|
||||||
|
return moreSevereConnectionSystemMemberState(existing.State, candidate.State)
|
||||||
|
case existing.LastSeen == nil:
|
||||||
|
return candidate.State
|
||||||
|
case candidate.LastSeen == nil:
|
||||||
|
return existing.State
|
||||||
|
case candidate.LastSeen.After(*existing.LastSeen):
|
||||||
|
return candidate.State
|
||||||
|
case existing.LastSeen.After(*candidate.LastSeen):
|
||||||
|
return existing.State
|
||||||
|
default:
|
||||||
|
return moreSevereConnectionSystemMemberState(existing.State, candidate.State)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func moreSevereConnectionSystemMemberState(
|
func moreSevereConnectionSystemMemberState(
|
||||||
left ConnectionState,
|
left ConnectionState,
|
||||||
right ConnectionState,
|
right ConnectionState,
|
||||||
|
|||||||
@@ -337,6 +337,59 @@ func TestMergeConnectionSystemMembersKeepsMostSevereState(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A lapsed projection (orphaned unified-resource entry after a
|
||||||
|
// remove/re-enroll cycle) must not pair its severity with the live plane's
|
||||||
|
// fresh heartbeat: that rendered members as "Stale · 5s ago". The plane with
|
||||||
|
// the newer LastSeen decides the state; severity still wins when neither side
|
||||||
|
// carries evidence or the timestamps tie. Refs #1728.
|
||||||
|
func TestMergeConnectionSystemMembersFresherEvidenceDecidesState(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
fresh := now.Add(-5 * time.Second)
|
||||||
|
lapsed := now.Add(-3 * 24 * time.Hour)
|
||||||
|
|
||||||
|
liveNode := ConnectionSystemMember{
|
||||||
|
ID: "node-minipc",
|
||||||
|
Name: "minipc",
|
||||||
|
State: ConnectionStateActive,
|
||||||
|
LastSeen: &fresh,
|
||||||
|
Primary: true,
|
||||||
|
}
|
||||||
|
orphanResource := ConnectionSystemMember{
|
||||||
|
ID: "resource-node-minipc",
|
||||||
|
Name: "minipc",
|
||||||
|
State: ConnectionStateStale,
|
||||||
|
LastSeen: &lapsed,
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, merged := range map[string]ConnectionSystemMember{
|
||||||
|
"live-first": mergeConnectionSystemMembers(liveNode, orphanResource),
|
||||||
|
"orphan-first": mergeConnectionSystemMembers(orphanResource, liveNode),
|
||||||
|
} {
|
||||||
|
if merged.State != ConnectionStateActive {
|
||||||
|
t.Fatalf("%s: merged state = %q, want %q", name, merged.State, ConnectionStateActive)
|
||||||
|
}
|
||||||
|
if merged.LastSeen == nil || !merged.LastSeen.Equal(fresh) {
|
||||||
|
t.Fatalf("%s: merged lastSeen = %v, want %v", name, merged.LastSeen, fresh)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An evidence-bearing severe plane still wins over a plane that has no
|
||||||
|
// evidence at all, and over an equally recent healthy plane.
|
||||||
|
unreachable := ConnectionSystemMember{
|
||||||
|
Name: "minipc",
|
||||||
|
State: ConnectionStateUnreachable,
|
||||||
|
LastSeen: &lapsed,
|
||||||
|
}
|
||||||
|
noEvidence := ConnectionSystemMember{Name: "minipc", State: ConnectionStateActive}
|
||||||
|
if merged := mergeConnectionSystemMembers(noEvidence, unreachable); merged.State != ConnectionStateUnreachable {
|
||||||
|
t.Fatalf("no-evidence merge state = %q, want %q", merged.State, ConnectionStateUnreachable)
|
||||||
|
}
|
||||||
|
tied := ConnectionSystemMember{Name: "minipc", State: ConnectionStateActive, LastSeen: &lapsed}
|
||||||
|
if merged := mergeConnectionSystemMembers(tied, unreachable); merged.State != ConnectionStateUnreachable {
|
||||||
|
t.Fatalf("tied-recency merge state = %q, want %q", merged.State, ConnectionStateUnreachable)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConnectionSystemMemberUsesDisplayNameWithoutChangingMemberKey(t *testing.T) {
|
func TestConnectionSystemMemberUsesDisplayNameWithoutChangingMemberKey(t *testing.T) {
|
||||||
resource := unified.Resource{
|
resource := unified.Resource{
|
||||||
ID: "resource-node",
|
ID: "resource-node",
|
||||||
|
|||||||
Reference in New Issue
Block a user