diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 2a219c14f4..8112663fdc 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -6670,11 +6670,43 @@ separate. still take it from each other on alternate rebuilds, as before. Evidence a later record pass of the same rebuild merges into an already-joined row reaches neither side's pin. -- Fold history is registry-local: resource seeding and overlay clones do not - carry it. A reference to the folded resource's canonical ID, such as an - availability check configured against an agent later linked into its - guest, does not resolve to the primary, so the check loses its target in - the broadcast as it already did in the resources API. +- A folded resource's canonical ID resolves to the primary holding it. The + registry indexes the folded IDs named by the merged resource's link fold + record (`linkFolds`, `recordManualLinkFold`), which covers IDs folded along + a chain of links, both when a link applies and when resource seeding + copies the record in. The monitor's rebuild, a registry seeded from its listing (the resources API, + which then replays availability checks) and a read-state overlay therefore + resolve them alike. The index follows the holding rows, so a holder that a + later link merges away or that is re-keyed in place keeps the answer + consistent. An availability check configured against an agent later + linked into its guest lands on the merged row, as a check configured + against the primary would. "A manual link keeps only the primary's own + checks" still holds: the merge never carries the folded resource's + projected checks; only the check's own reference resolves again. + `ResolveReferenceID` and `GetByReference` follow the fold as well, as the + folded agent's alert reference (`agent:`) already did through the + merged row's canonical aliases. Alert intent, operator-state reads and + writes through the resources API and resource lookups by the folded ID + (`GET /api/resources/{id}`) therefore reach the merged row. Patrol's finding auto-acknowledgement + still reads a row stored under the exact reference first, so an + operator-state row kept under the folded ID applies there. History does + not follow the fold: a history binding persists, and + `expandHistoryAliases` walks bindings in both directions, so binding a + folded ID would join the two journals for good. Folded IDs are never + `SupersededCanonicalIDs`, which alert-configuration and availability-link + migrations rewrite permanently; the stored link stays as the operator + wrote it. A live row with a folded ID answers for itself. Two rows holding + one folded ID resolve to neither, without falling through to weaker + matches such as a hostname alias. Folds ride in-memory clones only; a + serialized listing loses them, as it loses source parents. + `availability_link_test.go` pins both link directions through two + rebuilds in all three registries + (`TestAvailabilityLinkToALinkFoldedResourceFollowsItsPrimary`), a chain + (`TestAvailabilityLinkFollowsChainedLinkFolds`) and the index cases + (`TestLinkFoldIndexFollowsItsHolders`). + `TestResourceUnlinkSplitsManuallyLinkedPair` pins the API lookup: the + folded agent's ID answers with the VM while linked and with the agent's own + row once unlinked. `registry_merge_policy_test.go` pins the record links in both directions through two rebuilds and the live supplemental refresh, and checks that a diff --git a/internal/api/resourceapi/resources_unlink_test.go b/internal/api/resourceapi/resources_unlink_test.go index 2f27d0a671..37f6acd07e 100644 --- a/internal/api/resourceapi/resources_unlink_test.go +++ b/internal/api/resourceapi/resources_unlink_test.go @@ -99,10 +99,16 @@ func TestResourceUnlinkSplitsManuallyLinkedPair(t *testing.T) { if !ok { t.Fatalf("%s: VM %s missing from REST", step, vmID) } - _, agentListed := get(agentID) + // A lookup by the folded agent's ID answers with the row holding it, + // so the agent is listed only when its own row answers. + byAgentID, found := get(agentID) + agentListed := found && byAgentID.ID == agentID if wantLinked && (vm.Agent == nil || agentListed) { t.Fatalf("%s: want agent folded into the VM, VM agent facet=%v agent row listed=%v", step, vm.Agent != nil, agentListed) } + if wantLinked && byAgentID.ID != vmID { + t.Fatalf("%s: lookup by folded agent %s answered %q (found %v), want the VM %s", step, agentID, byAgentID.ID, found, vmID) + } if !wantLinked && (vm.Agent != nil || !agentListed) { t.Fatalf("%s: want VM and agent apart, VM agent facet=%v agent row listed=%v", step, vm.Agent != nil, agentListed) } diff --git a/internal/unifiedresources/availability_link_test.go b/internal/unifiedresources/availability_link_test.go index fcb21d0289..e44edd3073 100644 --- a/internal/unifiedresources/availability_link_test.go +++ b/internal/unifiedresources/availability_link_test.go @@ -1,9 +1,11 @@ package unifiedresources import ( + "slices" "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" ) @@ -792,3 +794,294 @@ func TestProxmoxGuestAvailabilityLinkFollowsRetiredIDs(t *testing.T) { }) } } + +// assertAvailabilityCheckAttachedTo checks that the configured check targetID +// is projected onto resourceID by its explicit link and that the check's own +// endpoint row points there. +func assertAvailabilityCheckAttachedTo(t *testing.T, stage string, rr *ResourceRegistry, targetID, resourceID string) { + t.Helper() + target, ok := rr.Get(resourceID) + if !ok { + t.Fatalf("%s: check target %s missing from %v", stage, resourceID, resourceIDs(rr.List())) + } + attached := false + for _, check := range AvailabilityChecksForResource(*target) { + if check.TargetID == targetID { + attached = check.CorrelationState == AvailabilityCorrelationAttached && check.CorrelationRule == "explicit_resource_link" + } + } + if !attached { + t.Fatalf("%s: %s checks = %+v, want %s attached by its explicit link", stage, resourceID, AvailabilityChecksForResource(*target), targetID) + } + endpoint := availabilityEndpointByTarget(t, rr, targetID) + for _, relationship := range endpoint.Relationships { + if relationship.Type == RelChecks && relationship.TargetID == resourceID { + return + } + } + t.Fatalf("%s: endpoint relationships = %+v, want a checks edge to %s", stage, endpoint.Relationships, resourceID) +} + +func linkedGuestVMRecord(now time.Time) IngestRecord { + return IngestRecord{ + SourceID: "vc-1:vm:vm-42", + Resource: Resource{ + Type: ResourceTypeVM, + Technology: "vmware", + Name: "app-guest", + Status: StatusOnline, + LastSeen: now, + VMware: &VMwareData{ConnectionID: "vc-1", ManagedObjectID: "vm-42", EntityType: "vm"}, + }, + Identity: ResourceIdentity{Hostnames: []string{"app-guest"}}, + } +} + +func linkedGuestCheck(t *testing.T, targetID, address, linkedResourceID string, now time.Time) IngestRecord { + t.Helper() + return availabilityProbeRecord(targetID, address, &AvailabilityData{ + LinkedResourceID: linkedResourceID, + Address: address, + Protocol: "icmp", + Enabled: true, + Available: true, + LastChecked: &now, + Evidence: availabilityProbeEvidence(t, targetID, now), + }) +} + +// An operator link folds the agent inside a vSphere VM into the VM, so the +// agent's own canonical ID no longer names a row. A check configured against +// that ID watches the same machine and follows the agent to the merged row in +// every registry that lists it: the monitor's rebuild, a registry seeded from +// the monitor's listing that replays the checks (the resources API) and a +// read-state overlay. Reference reads (alert intent, operator state, API +// lookups) resolve the folded ID the same way. History does not, and the +// folded ID never becomes a superseded ID, which alert and availability +// migrations would turn into a permanent rewrite of the operator's config. +func TestAvailabilityLinkToALinkFoldedResourceFollowsItsPrimary(t *testing.T) { + now := time.Now().UTC() + snapshot := models.StateSnapshot{ + Hosts: []models.Host{{ID: "host-app-guest", Hostname: "app-guest", MachineID: "machine-app-guest", Status: "online", LastSeen: now}}, + LastUpdate: now, + } + agentID := MachineIdentityCanonicalID(ResourceTypeAgent, "machine-app-guest") + checks := []IngestRecord{linkedGuestCheck(t, "probe-guest", "192.0.2.60", agentID, now)} + records := map[DataSource][]IngestRecord{ + SourceVMware: {linkedGuestVMRecord(now)}, + SourceAvailability: checks, + } + + for _, agentPrimary := range []bool{false, true} { + name := "vm-primary" + if agentPrimary { + name = "agent-primary" + } + t.Run(name, func(t *testing.T) { + store := NewMemoryStore() + adapter := NewMonitorAdapter(NewRegistry(store)) + adapter.PopulateSnapshotAndSupplemental(snapshot, records) + assertAvailabilityCheckAttachedTo(t, "unlinked", adapter.currentRegistry(), "probe-guest", agentID) + + var vmID string + for _, resource := range adapter.GetAll() { + if resource.VMware != nil { + vmID = resource.ID + } + } + if vmID == "" { + t.Fatalf("unlinked estate = %v, want the vSphere VM", resourceIDs(adapter.GetAll())) + } + primaryID := vmID + if agentPrimary { + primaryID = agentID + } + if err := store.AddLink(ResourceLink{ResourceA: vmID, ResourceB: agentID, PrimaryID: primaryID}); err != nil { + t.Fatalf("add link: %v", err) + } + + // Pins persist on the first rebuild; damage from them shows on the second. + for i, stage := range []string{"rebuild", "second rebuild"} { + next := snapshot + next.LastUpdate = now.Add(time.Duration(i+1) * time.Second) + adapter.PopulateSnapshotAndSupplemental(next, records) + if _, listed := adapter.currentRegistry().Get(agentID); listed { + t.Fatalf("%s: agent %s still listed beside the VM it is linked into", stage, agentID) + } + + rest := NewRegistry(store) + rest.IngestResources(adapter.GetAll()) + rest.IngestRecords(SourceAvailability, checks) + overlay, ok := ReadStateWithRecords(adapter, SourceAvailability, checks).(*MonitorAdapter) + if !ok { + t.Fatalf("%s: overlay is not a monitor adapter", stage) + } + + for _, view := range []struct { + name string + rr *ResourceRegistry + }{ + {"monitor", adapter.currentRegistry()}, + {"resources API", rest}, + {"overlay", overlay.currentRegistry()}, + } { + label := stage + ", " + view.name + assertAvailabilityCheckAttachedTo(t, label, view.rr, "probe-guest", vmID) + // The agent's alert reference already reached the merged + // row; its canonical ID must agree. + for _, ref := range []string{agentID, "agent:host-app-guest"} { + if got, ok := view.rr.ResolveReferenceID(ref); !ok || got != vmID { + t.Fatalf("%s: ResolveReferenceID(%s) = %q, %t, want the VM %s", label, ref, got, ok, vmID) + } + } + // History bindings persist and join both journals in either + // direction, so they would outlive the link. + if got, claimed := view.rr.resolveHistoryReference(agentID); got != "" || claimed { + t.Fatalf("%s: history reference %s resolved to %q (claimed %t), want it left to its own journal", label, agentID, got, claimed) + } + vm, _ := view.rr.Get(vmID) + if slices.Contains(vm.SupersededCanonicalIDs, agentID) || + (vm.Canonical != nil && slices.Contains(vm.Canonical.SupersededIDs, agentID)) { + t.Fatalf("%s: folded agent %s is listed as superseded by the VM", label, agentID) + } + } + } + }) + } +} + +// An old enrollment of a guest linked into its current agent, and that agent +// linked into its VM: every ID folded along the chain resolves to the final +// primary, in the monitor and in a registry seeded from it. +func TestAvailabilityLinkFollowsChainedLinkFolds(t *testing.T) { + now := time.Now().UTC() + snapshot := models.StateSnapshot{ + Hosts: []models.Host{ + {ID: "host-app-guest", Hostname: "app-guest", MachineID: "machine-app-guest", Status: "online", LastSeen: now}, + {ID: "host-app-guest-old", Hostname: "app-guest-old", MachineID: "machine-app-guest-old", Status: "online", LastSeen: now}, + }, + LastUpdate: now, + } + agentID := MachineIdentityCanonicalID(ResourceTypeAgent, "machine-app-guest") + oldAgentID := MachineIdentityCanonicalID(ResourceTypeAgent, "machine-app-guest-old") + checks := []IngestRecord{linkedGuestCheck(t, "probe-old", "192.0.2.61", oldAgentID, now)} + records := map[DataSource][]IngestRecord{ + SourceVMware: {linkedGuestVMRecord(now)}, + SourceAvailability: checks, + } + + unlinked := NewMonitorAdapter(NewRegistry(nil)) + unlinked.PopulateSnapshotAndSupplemental(snapshot, records) + var vmID string + for _, resource := range unlinked.GetAll() { + if resource.VMware != nil { + vmID = resource.ID + } + } + listed := resourceIDs(unlinked.GetAll()) + if vmID == "" || !slices.Contains(listed, agentID) || !slices.Contains(listed, oldAgentID) { + t.Fatalf("unlinked estate = %v, want the VM and both agents", listed) + } + + store := NewMemoryStore() + for _, link := range []ResourceLink{ + {ResourceA: agentID, ResourceB: oldAgentID, PrimaryID: agentID}, + {ResourceA: vmID, ResourceB: agentID, PrimaryID: vmID}, + } { + if err := store.AddLink(link); err != nil { + t.Fatalf("add link: %v", err) + } + } + adapter := NewMonitorAdapter(NewRegistry(store)) + adapter.PopulateSnapshotAndSupplemental(snapshot, records) + listed = resourceIDs(adapter.GetAll()) + if slices.Contains(listed, agentID) || slices.Contains(listed, oldAgentID) { + t.Fatalf("linked estate = %v, want both agents folded into the VM %s", listed, vmID) + } + + rest := NewRegistry(store) + rest.IngestResources(adapter.GetAll()) + rest.IngestRecords(SourceAvailability, checks) + for _, view := range []struct { + name string + rr *ResourceRegistry + }{{"monitor", adapter.currentRegistry()}, {"resources API", rest}} { + assertAvailabilityCheckAttachedTo(t, view.name, view.rr, "probe-old", vmID) + for _, folded := range []string{agentID, oldAgentID} { + if got, ok := view.rr.ResolveReferenceID(folded); !ok || got != vmID { + t.Fatalf("%s: ResolveReferenceID(%s) = %q, %t, want the VM %s", view.name, folded, got, ok, vmID) + } + } + } +} + +// The fold index follows the rows holding each folded ID. A seeded listing can +// carry one folded ID on two rows, or on a row while a live row has that ID. A +// live row answers for itself; two holders resolve to nothing, without falling +// through to weaker matches such as a hostname alias, so an explicit check link +// stays fail-closed. Holders a link later merges, and a holder re-keyed in +// place, keep resolving. +func TestLinkFoldIndexFollowsItsHolders(t *testing.T) { + now := time.Now().UTC() + vm := func(id string, folded ...string) Resource { + folds := make([]ManualLinkFold, 0, len(folded)) + for _, foldedID := range folded { + folds = append(folds, ManualLinkFold{HolderID: id, FoldedID: foldedID, Sources: []DataSource{SourceAgent}}) + } + return Resource{ + ID: id, Type: ResourceTypeVM, Name: id, Status: StatusOnline, LastSeen: now, + Sources: []DataSource{SourceVMware}, linkFolds: folds, + } + } + + t.Run("ambiguous holders fail closed", func(t *testing.T) { + rr := NewRegistry(nil) + rr.IngestResources([]Resource{ + vm("vm-a", "agent-shared", "agent-live"), + vm("vm-b", "agent-shared"), + {ID: "agent-live", Type: ResourceTypeAgent, Name: "agent-live", Status: StatusOnline, LastSeen: now, Sources: []DataSource{SourceAgent}}, + { + ID: "vm-c", Type: ResourceTypeVM, Name: "vm-c", Status: StatusOnline, LastSeen: now, + Sources: []DataSource{SourceVMware}, Identity: ResourceIdentity{Hostnames: []string{"agent-shared"}}, + }, + }) + if got, ok := rr.ResolveReferenceID("agent-shared"); ok { + t.Fatalf("ResolveReferenceID(agent-shared) = %q, want no answer while two rows hold it", got) + } + if got, ok := rr.ResolveReferenceID("agent-live"); !ok || got != "agent-live" { + t.Fatalf("ResolveReferenceID(agent-live) = %q, %t, want the live row itself", got, ok) + } + rr.IngestRecords(SourceAvailability, []IngestRecord{linkedGuestCheck(t, "probe-shared", "192.0.2.62", "agent-shared", now)}) + endpoint := availabilityEndpointByTarget(t, rr, "probe-shared") + if endpoint.Availability.CorrelationState != AvailabilityCorrelationUnresolved { + t.Fatalf("ambiguous folded link correlation = %+v, want unresolved", endpoint.Availability) + } + }) + + t.Run("holders merged by a link resolve to the survivor", func(t *testing.T) { + store := NewMemoryStore() + if err := store.AddLink(ResourceLink{ResourceA: "vm-a", ResourceB: "vm-b", PrimaryID: "vm-a"}); err != nil { + t.Fatalf("add link: %v", err) + } + rr := NewRegistry(store) + rr.IngestResources([]Resource{vm("vm-a", "agent-shared"), vm("vm-b", "agent-shared")}) + for _, ref := range []string{"agent-shared", "vm-b"} { + if got, ok := rr.ResolveReferenceID(ref); !ok || got != "vm-a" { + t.Fatalf("ResolveReferenceID(%s) = %q, %t, want vm-a", ref, got, ok) + } + } + }) + + t.Run("re-keyed holder keeps its folds", func(t *testing.T) { + rr := NewRegistry(nil) + disk := vm("disk-old", "disk-folded") + disk.Type = ResourceTypePhysicalDisk + rr.IngestResources([]Resource{disk}) + rr.mu.Lock() + rr.rekeyPhysicalDiskLocked(rr.resources["disk-old"], "disk-new") + rr.mu.Unlock() + if got, ok := rr.ResolveReferenceID("disk-folded"); !ok || got != "disk-new" { + t.Fatalf("ResolveReferenceID(disk-folded) = %q, %t, want the re-keyed holder disk-new", got, ok) + } + }) +} diff --git a/internal/unifiedresources/registry.go b/internal/unifiedresources/registry.go index 5e7f4f2024..ca34db477c 100644 --- a/internal/unifiedresources/registry.go +++ b/internal/unifiedresources/registry.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "log" + "slices" "sort" "strings" "sync" @@ -95,10 +96,15 @@ type ResourceRegistry struct { exclusions map[string]struct{} identityPins *identityPinIndex pbsBackups []models.PBSBackup - // linkMergedIDs holds the resources a manual link folded into their - // primary. They are still observed, so identity succession treats them - // as live. - linkMergedIDs map[string]struct{} + // linkFoldIndex maps each canonical ID a manual link folded into another + // resource to the rows that took it in (Resource.linkFolds). Folded + // resources are still observed, so identity succession treats them as + // live, and references to them resolve to the row holding them. Holders + // carry their folds, so a registry seeded from another registry's listing + // indexes the same ones. Entries are rows, not IDs, and are checked on + // read (linkFoldHoldersLocked): a holder folded away, replaced or re-keyed + // needs no bookkeeping. + linkFoldIndex map[string][]*Resource // linkOwnPins holds the identity pin each side of a manual link derives // from its own sources, captured before the link merged them. A nil // value records a side with no pinnable identity. PersistIdentityPins @@ -800,8 +806,59 @@ func (rr *ResourceRegistry) canonicalIDObservedLocked(id string) bool { if _, live := rr.resources[id]; live { return true } - _, folded := rr.linkMergedIDs[id] - return folded + return len(rr.linkFoldHoldersLocked(id)) > 0 +} + +// indexLinkFoldsLocked records that holder took in the canonical IDs its +// link folds name (Resource.linkFolds, recordManualLinkFold). +func (rr *ResourceRegistry) indexLinkFoldsLocked(holder *Resource) { + if holder == nil || len(holder.linkFolds) == 0 { + return + } + if rr.linkFoldIndex == nil { + rr.linkFoldIndex = make(map[string][]*Resource) + } + for _, fold := range holder.linkFolds { + if !slices.Contains(rr.linkFoldIndex[fold.FoldedID], holder) { + rr.linkFoldIndex[fold.FoldedID] = append(rr.linkFoldIndex[fold.FoldedID], holder) + } + } +} + +// holdsLinkFold reports whether a resource's link folds name foldedID. +func holdsLinkFold(resource *Resource, foldedID string) bool { + for _, fold := range resource.linkFolds { + if fold.FoldedID == foldedID { + return true + } + } + return false +} + +// linkFoldHoldersLocked lists the live rows that hold a canonical ID an +// operator link folded into them. More than one is ambiguous. +func (rr *ResourceRegistry) linkFoldHoldersLocked(ref string) []string { + ref = CanonicalResourceID(ref) + var holders []string + for _, holder := range rr.linkFoldIndex[ref] { + if rr.resources[holder.ID] != holder || !holdsLinkFold(holder, ref) || + slices.Contains(holders, holder.ID) { + continue + } + holders = append(holders, holder.ID) + } + return holders +} + +// linkFoldHolderLocked resolves a canonical ID an operator link folded into +// another resource to the row holding it. ambiguous reports two holders: the +// reference must then resolve to nothing, not fall through to weaker matches. +func (rr *ResourceRegistry) linkFoldHolderLocked(ref string) (holderID string, ambiguous bool) { + holders := rr.linkFoldHoldersLocked(ref) + if len(holders) != 1 { + return "", len(holders) > 1 + } + return holders[0], false } // applyRecordSuccessions re-keys operator-owned store rows from canonical IDs @@ -901,6 +958,7 @@ func (rr *ResourceRegistry) ingestResources(resources []Resource, thresholds map rr.seedSourceMappingsFromResourceLocked(rr.resources[resourceID]) if resource := rr.resources[resourceID]; resource != nil { rr.indexSupersededCanonicalIDsLocked(resourceID, resource.SupersededCanonicalIDs) + rr.indexLinkFoldsLocked(resource) rr.indexPhysicalDiskHardwareLocked(resource) } } @@ -1537,6 +1595,11 @@ func (rr *ResourceRegistry) resolveReferenceIDLocked(ref string) (string, bool) if resolvedID := rr.supersededResourceIDLocked(ref); rr.resources[resolvedID] != nil { return resolvedID, false } + // A resource an operator link folded into another is that row now, as + // its agent alert references ("agent:") already are. + if resolvedID, ambiguous := rr.linkFoldHolderLocked(ref); resolvedID != "" || ambiguous { + return resolvedID, false + } if resolvedID := rr.uniqueSourceResourceIDLocked(ref); rr.resources[resolvedID] != nil { return resolvedID, false } @@ -3613,10 +3676,12 @@ func (rr *ResourceRegistry) resolveAvailabilityLinkedResource(ref string, incomi return "" } - // References persisted under a retired canonical ID or under a - // node-scoped guest source ID follow the resource across identity eras - // and live migrations. These arms resolve provider-declared persistence - // keys, not display aliases, so the explicit link stays fail-closed. + // References persisted under a retired canonical ID, under a resource an + // operator link folded into another, or under a node-scoped guest source + // ID follow the resource across identity eras, links and live + // migrations. These arms resolve provider-declared persistence keys and + // operator links, not display aliases, so the explicit link stays + // fail-closed. eligible := func(candidateID string) string { candidateID = CanonicalResourceID(candidateID) existing := rr.resources[candidateID] @@ -3628,6 +3693,9 @@ func (rr *ResourceRegistry) resolveAvailabilityLinkedResource(ref string, incomi if candidateID := eligible(rr.supersededResourceIDLocked(exactID)); candidateID != "" { return candidateID } + if holderID, ambiguous := rr.linkFoldHolderLocked(exactID); holderID != "" || ambiguous { + return eligible(holderID) + } if candidateID := eligible(rr.uniqueSourceResourceIDLocked(ref)); candidateID != "" { return candidateID } @@ -4851,10 +4919,10 @@ func (rr *ResourceRegistry) applyManualLinks(thresholds map[DataSource]time.Dura rr.mergeResourceData(primary, other, thresholds) recordManualLinkFold(primary, primaryID, other, otherID) delete(rr.resources, otherID) - if rr.linkMergedIDs == nil { - rr.linkMergedIDs = make(map[string]struct{}) - } - rr.linkMergedIDs[otherID] = struct{}{} + // The fold record names every ID along a chain of links, so the chain + // resolves to its last primary; the folded row's own index entries + // lapse on read now that it has left the registry. + rr.indexLinkFoldsLocked(primary) rr.updateSourceMappings(otherID, primaryID) } }