From f4c104621580024ea79b321a61c8c9b5879aa4a9 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:07:06 +0100 Subject: [PATCH] Record Docker host, service and sub-resource alert history Alert lifecycle events resolve to canonical resources since Proxmox and agent references gained durable resolution, but Docker host alerts ("docker:"), Swarm service alerts ("docker:/service/") and sub-resource alerts such as ZFS pool and device health ("/zfs-pool:") or host filesystem usage ("agent:/disk:var") were still journaled under references no resource answers to. Resource facets, timelines, resource intelligence and Assistant context read history by canonical ID, so they never saw them. Docker host and service references now resolve through the registry's own Docker source identities. A service reference must carry the service's real ID in the host's Swarm cluster, from any manager that reports it; services without an ID are referenced by name and never bind. Container names and shortened IDs still bind nothing and are no longer retried against later inventory. Sub-resource references bind to the owner whose durable ID they carry: the storage for ZFS pools and devices, the Unraid array storage, and the host for filesystem, SMART disk, disk temperature, RAID and custom-sensor alerts. Mount and kernel device labels can name a different disk after a reboot, so they never bind a physical disk. Any other shape stays unbound. On a mock stack none of the Docker host, Swarm service, ZFS and agent disk alert rows were readable through their owner's timeline API before this change; all of them are after it, including rows journaled earlier. The mock's shortened container ID rows stay unbound. --- .../v6/internal/subsystems/monitoring.md | 14 + .../internal/subsystems/unified-resources.md | 46 +++- .../monitoring/monitor_alert_handling_test.go | 61 +++++ internal/unifiedresources/history_identity.go | 196 ++++++++++++-- .../unifiedresources/history_identity_test.go | 243 ++++++++++++++++++ internal/unifiedresources/monitor_adapter.go | 26 +- 6 files changed, 546 insertions(+), 40 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 01286836ea..e1b8d4a5e7 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1052,6 +1052,20 @@ replay stays idempotent. The real alert-manager callback path is covered by `TestProxmoxAlertTimelineUsesCanonicalHistoryIdentity` in `internal/monitoring/monitor_alert_handling_test.go`. +Docker host and Swarm service alert lifecycle events, and sub-resource alert +events (ZFS pools and devices, host filesystems, disks, RAID arrays, sensors +and the Unraid array), pass through the same writer, which binds them to the +Docker host, Swarm service, storage or host named in the unified-resources +history identity clause. Container names and shortened container IDs still +bind nothing. `CheckDockerHost` gives a container without an ID its host's +reference, so that container's events join the host's history. A producer +that changes one of these reference shapes must change +`dockerHostHistoryReference` or `historySubResourceOwner` with it. +`TestOwnerAlertTimelinesUseCanonicalHistoryIdentity` in +`internal/monitoring/monitor_alert_handling_test.go` pins the Docker host, +Swarm service, ZFS pool and device, and host filesystem shapes against the +real alert manager. + TrueNAS native alert projection preserves the trimmed, uppercase provider level in ResourceIncident.NativeSeverity. INFO and NOTICE retain the same canonical monitor risk; consumers must not lose their distinct actionability when projecting provider evidence. Native CRITICAL, ALERT, and EMERGENCY all project to canonical critical severity; EMERGENCY must not be discarded as unknown or make a still-active condition appear recovered. WARNING remains warning, and INFO and NOTICE remain informational at this projection boundary. Verification: `TestIncidentProjectionPreservesNativeSeverity` in `internal/truenas/provider_pool_health_contract_test.go` covers all seven native levels and case/whitespace normalization. `TestTrueNASNativeSeverityDispatch` in `internal/alerts/truenas_native_dispatch_test.go` verifies downstream INFO suppression, NOTICE preservation, notification severity, duplicate-poll retention, and confirmed recovery callback identity. The TrueNAS lifecycle tests in `internal/alerts/unified_incidents_test.go` require repeated EMERGENCY evidence to interrupt recovery confirmation. These are fixture-based projection and manager checks, not appliance ingestion or external notification-provider receipt proof. diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 8b80254b40..1b1e4d58d9 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -5846,10 +5846,12 @@ and storage IDs), a node-scoped Proxmox guest reference after migration, or `ResolveReferenceID` used by alert policy, so a resource named like a system reference cannot capture its events. A reference two resources answer to binds to neither: its event keeps its own reference and creates or changes no -binding, though an earlier binding still covers that reference on read. A +binding, though an earlier binding still covers that reference on read. Like +any unbound reference it is retried, and once a later generation resolves it +to one resource it binds there, replacing an earlier binding. A reference no resource answers to follows its retained binding after inventory removal, or keeps its -own history. Docker references never take this path. Alert journal references +own history. Docker container references never take this path. Alert journal references without a binding (read once per process, plus any written before inventory named their resource) are retried on each published registry generation for `legacyHistoryBindWindow`. Bound rows keep their recorded resource ID, so a @@ -5861,6 +5863,46 @@ Assistant and Patrol resource contexts read it from the store. Proof: `TestHistoryIdentityMonitorAdapterResolvesProxmoxAlertReferences` and `TestHistoryIdentityBindsLegacyAlertRowsFromRegistryGenerations` in `internal/unifiedresources/history_identity_test.go`. +Docker host alerts (`docker:`) and Swarm service alerts +(`docker:/service/`) resolve only through the registry's +Docker source identities: the host ID is the Docker host's source ID, and a +service reference names the service with that exact ID in the host's Swarm +cluster, whichever manager reported it. A hostname in the host position is +only ever looked up as a Docker host source ID. A service without an ID is +referenced by its normalized name in the same place, so while its cluster has +such a service every service reference there is a conflict. Alert references +do not record whether they came from an ID, so a row journaled by an ID-less +service can still join another service's history if a later generation or +event binds the same reference; Swarm always reports service IDs, so that needs a malformed +report naming a service exactly like another service's ID. Hostless +`docker-service:` references, container names and shortened container +IDs never bind and are never retried against later inventory. A container +without an ID alerts under its host's reference (`alerts.DockerResourceID`), so +its events join that host's history, never another container's. +Sub-resource alert references name no resource of their own and bind to the +owner whose durable ID they carry (`historySubResourceOwner`): ZFS pool and +device alerts (`/zfs-pool:[/device:]`) to the +storage, the Unraid array alert (`agent:/storage:unraid-array`) to +the array storage with that agent source ID, and host filesystem, SMART disk, +disk temperature, RAID and custom-sensor alerts (`agent:/disk:`, +`/disk_temp:`, `/raid:`, `/custom:`) to the host. The owner reference must +resolve to exactly one resource of the expected type; an ambiguous or +incompatible owner is a conflict, so the event keeps its own reference, +follows no retained binding and is retried. Pool, mount and kernel device labels are names, +and a device name can denote a different disk after a reboot, so these alerts +never bind a physical disk. Any other sub-resource shape stays unbound until +its producer and this list change together. The Docker, storage and platform +host-row drawers do not request per-resource history, so these bindings reach +the facets, timeline and intelligence APIs and Assistant resource context, not +a drawer. Patrol's scoped change feed matches the stored resource ID without +expanding bindings, so it includes events recorded under the owner's +canonical ID but not rows journaled before their binding. +Proof: `TestHistoryIdentityMonitorAdapterResolvesDockerHostAndServiceReferences` +and `TestHistoryIdentityMonitorAdapterResolvesSubResourceReferences` in +`internal/unifiedresources/history_identity_test.go`, and +`TestOwnerAlertTimelinesUseCanonicalHistoryIdentity` in +`internal/monitoring/monitor_alert_handling_test.go` against the real alert +producers. That same shared timeline vocabulary now includes the `activity` change kind for provider-read breadcrumbs such as VMware tasks and events, plus the `vmware_adapter` source-adapter token for canonical provenance drill-down. diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index bb65eb663a..a8b7454b6c 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -150,6 +150,67 @@ func TestProxmoxAlertTimelineUsesCanonicalHistoryIdentity(t *testing.T) { require.Equal(t, canonical["pve1"], changes[0].ResourceID) } +// Docker host, Swarm service and sub-resource alerts (a ZFS pool and device, +// a host filesystem) carry references the registry has no resource for. Their +// lifecycle must reach the owner's canonical history, so the producers' ref +// shapes are pinned here against the real alert manager. +func TestOwnerAlertTimelinesUseCanonicalHistoryIdentity(t *testing.T) { + store, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + manager := alerts.NewManagerWithDataDir(t.TempDir()) + t.Cleanup(manager.Stop) + config := manager.GetConfig() + config.Enabled = true + config.ActivationState = alerts.ActivationPending + config.TimeThresholds = map[string]int{"all": 0} + config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0}} + config.SuppressionWindow = 0 + manager.UpdateConfig(config) + now := time.Now() + dockerHost := models.DockerHost{ID: "dh-7f3a", Hostname: "tower", DisplayName: "tower", Status: "online", LastSeen: now, + Swarm: &models.DockerSwarmInfo{NodeID: "node-a", NodeRole: "manager", LocalState: "active", ControlAvailable: true, ClusterID: "swarm-1"}, + Services: []models.DockerService{{ID: "x7k2m9q4w1e8r5t3y6u0i2o4p", Name: "web", DesiredTasks: 2, RunningTasks: 0}}} + storage := models.Storage{ID: "lab-pve1-local-zfs", Name: "local-zfs", Node: "pve1", Instance: "lab", Type: "zfspool", Status: "available", Total: 100, Used: 10, Usage: 10, Enabled: true, Active: true, LastSeen: now, + ZFSPool: &models.ZFSPool{Name: "rpool", State: "DEGRADED", Status: "Degraded", Devices: []models.ZFSDevice{{Name: "sda2", Type: "disk", State: "FAULTED"}}}} + agentHost := models.Host{ID: "host-nas", Hostname: "nas", DisplayName: "nas", MachineID: "fedcba9876543210", Status: "online", LastSeen: now, + Disks: []models.Disk{{Mountpoint: "/var", Device: "/dev/sdb1", Type: "ext4", Total: 100, Used: 99, Free: 1, Usage: 99}}} + registry := unifiedresources.NewRegistry(store) + registry.IngestSnapshot(models.StateSnapshot{DockerHosts: []models.DockerHost{dockerHost}, Storage: []models.Storage{storage}, Hosts: []models.Host{agentHost}}) + monitor := &Monitor{alertManager: manager, resourceStore: unifiedresources.NewMonitorAdapter(registry)} + manager.SubscribeLifecycleCallback(monitor.handleAlertLifecycleEvent) + manager.CheckDockerHost(dockerHost) + manager.CheckStorage(storage) + manager.CheckHost(agentHost) + for range 3 { + manager.HandleDockerHostOffline(dockerHost) + } + + owner := map[string]string{} + for _, resource := range registry.List() { + owner[string(resource.Type)+"/"+resource.Name] = resource.ID + } + filters := unifiedresources.ResourceChangeFilters{Kinds: []unifiedresources.ChangeKind{unifiedresources.ChangeAlertFired}} + want := map[string]string{ + "docker-host-offline": owner["agent/tower"], + "docker-service-health": owner["docker-service/web"], + "zfs-pool-state": owner["storage/local-zfs"], + "zfs-device": owner["storage/local-zfs"], + "disk": owner["agent/nas"], + } + for alertType, canonicalID := range want { + require.NotEmpty(t, canonicalID, alertType) + changes, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, filters) + require.NoError(t, err) + found := false + for _, change := range changes { + require.Equal(t, canonicalID, change.ResourceID) + found = found || change.Metadata["alert_type"] == alertType + } + require.True(t, found, "%s alert missing from %s history: %+v", alertType, canonicalID, changes) + } +} + func TestMonitor_HandleAlertFired_Extra(t *testing.T) { // 1. Alert is nil m1 := &Monitor{} diff --git a/internal/unifiedresources/history_identity.go b/internal/unifiedresources/history_identity.go index 9dc69d886e..61ddc1ce1b 100644 --- a/internal/unifiedresources/history_identity.go +++ b/internal/unifiedresources/history_identity.go @@ -32,35 +32,180 @@ func legacyDockerHistoryIdentity(ref string) (sourceID, canonicalID string, ok b return sourceID, SourceSpecificID(ResourceTypeAppContainer, SourceDocker, sourceID), true } -// isDockerHistoryReference marks Docker alert source references, which only a -// complete container ID binds (legacyDockerHistoryIdentity). A container name -// or shortened ID must never join history through general reference matching. +// isDockerHistoryReference marks Docker alert source references, including the +// hostless "docker-service:" form. They never join history through +// general reference matching: only a Docker host, a Swarm service ID +// (dockerHostHistoryReference) or a complete container ID +// (legacyDockerHistoryIdentity) binds. func isDockerHistoryReference(ref string) bool { - return strings.HasPrefix(strings.TrimSpace(ref), "docker:") + ref = strings.TrimSpace(ref) + return strings.HasPrefix(ref, "docker:") || strings.HasPrefix(ref, "docker-service:") } -// resolveHistoryReference resolves a non-Docker alert reference to the -// resource whose history it joins. Only durable identities qualify: the -// canonical ID, a retired era, a source ID (Proxmox node, guest and storage -// IDs), a node-scoped Proxmox guest reference, or an agent alert reference -// ("agent:"). Names and hostnames never bind history, so a resource -// named like a system reference cannot capture its events. claimed reports -// that inventory answers to the reference, possibly ambiguously; an ambiguous -// reference resolves to nothing and must not follow a retained binding. +// isDockerNameHistoryReference marks Docker alert references that are not host +// or service references: container references, which only a complete +// container ID binds, and hostless service names. A name or shortened ID must +// never join history, not even through a retained binding. +func isDockerNameHistoryReference(ref string) bool { + _, _, ok := dockerHostHistoryReference(ref) + return isDockerHistoryReference(ref) && !ok +} + +// dockerHostHistoryReference parses the Docker host alert reference +// ("docker:", alerts.DockerResourceID without a container) and the +// Swarm service alert reference ("docker:/service/", +// alerts.DockerServiceResourceID). serviceID is empty for a host reference. +// "docker:unknown" is the shared fallback for alerts without a host. +func dockerHostHistoryReference(ref string) (hostID, serviceID string, ok bool) { + rest, found := strings.CutPrefix(strings.TrimSpace(ref), "docker:") + if !found { + return "", "", false + } + hostID, serviceID, isService := strings.Cut(rest, "/service/") + if hostID == "" || hostID == "unknown" || strings.TrimSpace(hostID) != hostID || strings.Contains(hostID, "/") { + return "", "", false + } + if isService && (serviceID == "" || strings.TrimSpace(serviceID) != serviceID || strings.Contains(serviceID, "/")) { + return "", "", false + } + return hostID, serviceID, true +} + +// dockerHistoryOwnerLocked resolves a Docker host or Swarm service alert +// reference through the registry's own Docker source identities. The host ID +// is the Docker host's source ID. A service reference names the service with +// that ID in the host's Swarm cluster, which every manager of the cluster +// reports under its own host ID. A service without an ID is referenced by its +// normalized name in the same place, so while the cluster has such a service +// every service reference there is a conflict. +func (rr *ResourceRegistry) dockerHistoryOwnerLocked(ref string) (resourceID string, conflict bool) { + hostID, serviceID, ok := dockerHostHistoryReference(ref) + if !ok { + return "", false + } + hostResourceID := rr.bySource[SourceDocker][hostID] + host := rr.resources[hostResourceID] + if host == nil || CanonicalResourceType(host.Type) != ResourceTypeAgent { + return "", false + } + if serviceID == "" { + return hostResourceID, false + } + if host.Docker == nil { + return "", false + } + cluster := dockerSwarmClusterKeyFromMeta(host.Docker.Swarm) + if cluster == "" { + return "", false + } + for _, other := range rr.resources { + if CanonicalResourceType(other.Type) == ResourceTypeDockerService && other.Docker != nil && + strings.TrimSpace(other.Docker.ServiceID) == "" && dockerSwarmClusterKeyFromMeta(other.Docker.Swarm) == cluster { + return "", true + } + } + serviceResourceID := rr.bySource[SourceDocker][normalizeSourceID(cluster+":service:"+serviceID)] + service := rr.resources[serviceResourceID] + if service == nil || CanonicalResourceType(service.Type) != ResourceTypeDockerService || + service.Docker == nil || strings.TrimSpace(service.Docker.ServiceID) != serviceID { + return "", false + } + return serviceResourceID, false +} + +// historySubResourceOwner names the owner of a sub-resource alert reference, +// one that names no resource of its own, and the type that owner must have. +// Only these producer shapes qualify, each carrying its owner's durable +// identity: +// - "/zfs-pool:[/device:]" (ZFS pool state, +// errors and device health) belongs to the storage. +// - "agent:/storage:" (Unraid array) is the agent source ID +// "/storage:" of the host's array storage. +// - "agent:/disk:", ".../disk_temp:", +// ".../raid:" and ".../custom:" belong to the host. +// +// Pool, mount and kernel device labels are names: a device name can belong to +// a different disk after a reboot, so these never bind a physical disk. +func historySubResourceOwner(ref string) (ownerRef string, ownerType ResourceType, ok bool) { + if rest, found := strings.CutPrefix(ref, "agent:"); found { + hostID, child, _ := strings.Cut(rest, "/") + kind, label, _ := strings.Cut(child, ":") + if hostID == "" || label == "" { + return "", "", false + } + switch kind { + case "storage": + return hostID + "/" + child, ResourceTypeStorage, true + case "disk", "disk_temp", "raid", "custom": + return "agent:" + hostID, ResourceTypeAgent, true + } + return "", "", false + } + if i := strings.LastIndex(ref, "/zfs-pool:"); i > 0 { + pool, device, hasDevice := strings.Cut(ref[i+len("/zfs-pool:"):], "/device:") + if pool == "" || strings.Contains(pool, "/") || (hasDevice && (device == "" || strings.Contains(device, "/"))) { + return "", "", false + } + return ref[:i], ResourceTypeStorage, true + } + return "", "", false +} + +// resolveHistoryReference resolves an alert reference to the resource whose +// history it joins. Only durable identities qualify: the canonical ID, a +// retired era, a source ID (Proxmox node, guest and storage IDs), a +// node-scoped Proxmox guest reference, an agent alert reference +// ("agent:"), a Docker host or Swarm service ID, or the owner of a +// sub-resource reference (historySubResourceOwner). Names and hostnames never +// bind history, so a resource named like a system reference cannot capture +// its events. claimed reports that inventory answers to the reference, +// possibly ambiguously; an ambiguous or conflicting reference resolves to +// nothing and must not follow a retained binding. func (rr *ResourceRegistry) resolveHistoryReference(ref string) (resourceID string, claimed bool) { ref = CanonicalResourceID(ref) - if rr == nil || ref == "" || isDockerHistoryReference(ref) { + if rr == nil || ref == "" { return "", false } rr.mu.RLock() defer rr.mu.RUnlock() + if isDockerHistoryReference(ref) { + resourceID, conflict := rr.dockerHistoryOwnerLocked(ref) + return resourceID, resourceID != "" || conflict + } + matches := rr.durableHistoryMatchesLocked(ref) + if ownerRef, ownerType, ok := historySubResourceOwner(ref); ok && len(matches) == 0 { + // The owner must resolve uniquely and have the expected type. An owner + // reference that names an ambiguous or incompatible resource is a + // conflict: the event keeps its own reference. + owners := rr.durableHistoryMatchesLocked(ownerRef) + for id := range owners { + if len(owners) != 1 || CanonicalResourceType(rr.resources[id].Type) != ownerType { + return "", true + } + matches[id] = struct{}{} + } + } + if len(matches) != 1 { + return "", len(matches) > 1 + } + for id := range matches { + resourceID = id + } + return resourceID, true +} + +// durableHistoryMatchesLocked lists the resources that answer to a reference +// by durable identity. +func (rr *ResourceRegistry) durableHistoryMatchesLocked(ref string) map[string]struct{} { + matches := make(map[string]struct{}, 1) if rr.resources[ref] != nil { - return ref, true + matches[ref] = struct{}{} + return matches } if id := rr.supersededResourceIDLocked(ref); rr.resources[id] != nil { - return id, true + matches[id] = struct{}{} + return matches } - matches := make(map[string]struct{}, 1) for _, mapping := range rr.bySource { if id := mapping[ref]; rr.resources[id] != nil { matches[id] = struct{}{} @@ -78,13 +223,7 @@ func (rr *ResourceRegistry) resolveHistoryReference(ref string) (resourceID stri } } } - if len(matches) != 1 { - return "", len(matches) > 1 - } - for id := range matches { - resourceID = id - } - return resourceID, true + return matches } // resourceHistoryIdentityWriter binds a source reference to the canonical @@ -187,11 +326,12 @@ func (b *legacyHistoryBackfill) add(ref string, now time.Time) { } // unboundHistoryReferences lists alert journal references without a binding. -// Docker references are excluded: the store migration binds full container -// IDs, and names or shortened IDs must never bind. +// Docker container references and hostless service names are excluded: the +// store migration binds full container IDs, and names or shortened IDs must +// never bind. func (s *SQLiteResourceStore) unboundHistoryReferences() ([]string, error) { rows, err := s.db.Query(`SELECT DISTINCT canonical_id FROM resource_changes - WHERE kind GLOB 'alert_*' AND canonical_id NOT GLOB 'docker:*' + WHERE kind GLOB 'alert_*' AND canonical_id NOT IN (SELECT source_id FROM resource_history_aliases) ORDER BY canonical_id`) if err != nil { @@ -204,7 +344,9 @@ func (s *SQLiteResourceStore) unboundHistoryReferences() ([]string, error) { if err := rows.Scan(&ref); err != nil { return nil, err } - refs = append(refs, ref) + if !isDockerNameHistoryReference(ref) { + refs = append(refs, ref) + } } return refs, rows.Err() } diff --git a/internal/unifiedresources/history_identity_test.go b/internal/unifiedresources/history_identity_test.go index 935e260fb4..4744e08503 100644 --- a/internal/unifiedresources/history_identity_test.go +++ b/internal/unifiedresources/history_identity_test.go @@ -333,3 +333,246 @@ func TestHistoryIdentityBindsLegacyAlertRowsFromRegistryGenerations(t *testing.T require.NoError(t, err) require.Equal(t, []string{"lab:pve1:999"}, refs) } + +func requireHistoryReferenceBindings(t *testing.T, store *SQLiteResourceStore, bound map[string]string, unbound []string) { + t.Helper() + byResource := make(map[string][]string) + for ref, canonicalID := range bound { + byResource[canonicalID] = append(byResource[canonicalID], "fired-"+ref) + id, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.True(t, found, ref) + require.Equal(t, canonicalID, id, ref) + } + for canonicalID, want := range byResource { + got, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 50, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + ids := make([]string, 0, len(got)) + for _, change := range got { + ids = append(ids, change.ID) + // A row journaled before its binding keeps its recorded reference. + require.Contains(t, []string{canonicalID, strings.TrimPrefix(change.ID, "fired-")}, change.ResourceID, change.ID) + } + require.ElementsMatch(t, want, ids, canonicalID) + } + for _, ref := range unbound { + _, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.False(t, found, ref) + got, err := store.GetRecentChanges(ref, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 1, ref) + require.Equal(t, ref, got[0].ResourceID, ref) + } +} + +// Docker host alerts reference the host's source ID ("docker:") and +// Swarm service alerts the service ID ("docker:/service/") from +// every manager that reports the service. Both join the canonical host and +// service history, also for a host merged into its agent's machine. Hostnames, +// service names, container names and shortened container IDs never bind, not +// even to the history of what they name, and a cluster with a service that +// lacks an ID, whose alerts carry its normalized name, binds no service. +func TestHistoryIdentityMonitorAdapterResolvesDockerHostAndServiceReferences(t *testing.T) { + store, err := NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + now := time.Now().UTC().Truncate(time.Second) + container := strings.Repeat("f", 64) + serviceID := "x7k2m9q4w1e8r5t3y6u0i2o4p" + otherServiceID := "q1w2e3r4t5y6u7i8o9p0a1s2d" + manager := func(id, hostname, cluster string, services ...models.DockerService) models.DockerHost { + return models.DockerHost{ID: id, Hostname: hostname, Status: "online", LastSeen: now, Services: services, + Swarm: &models.DockerSwarmInfo{NodeID: id + "-node", NodeRole: "manager", LocalState: "active", ControlAvailable: true, ClusterID: cluster}} + } + web := models.DockerService{ID: serviceID, Name: "web"} + // hostA runs on a machine whose agent also reports, so the two merge. + hostA := manager("dh-7f3a", "tower", "swarm-1", web) + hostA.AgentID, hostA.MachineID = "host-tower", "0123456789abcdef" + hostA.Containers = []models.DockerContainer{{ID: container, Name: "worker", State: "running"}} + hostB := manager("dh-91c0", "rack", "swarm-1", web) + // In swarm-2 a service without an ID is named like another service's ID. + hostC := manager("dh-c4d2", "edge", "swarm-2", models.DockerService{ID: otherServiceID, Name: "api"}, models.DockerService{Name: strings.ToUpper(otherServiceID)}) + // Rows journaled before this process started are bound by a later generation. + legacy := []string{"docker:dh-91c0", "docker:dh-91c0/service/" + serviceID, "docker:dh-7f3a/worker"} + for i, ref := range legacy { + require.NoError(t, store.RecordChange(ResourceChange{ID: "fired-" + ref, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Duration(i) * time.Second)})) + } + adapter := NewMonitorAdapter(NewRegistry(store)) + adapter.PopulateFromSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ID: "host-tower", Hostname: "tower", MachineID: "0123456789abcdef", Status: "online", LastSeen: now}}, + DockerHosts: []models.DockerHost{hostA, hostB, hostC}, + }) + registry := adapter.currentRegistry() + hostAID := registry.sourceResourceID(SourceDocker, "dh-7f3a") + hostBID := registry.sourceResourceID(SourceDocker, "dh-91c0") + serviceResourceID := historyIdentityResourceID(t, registry, ResourceTypeDockerService, "web") + containerID := registry.sourceResourceID(SourceDocker, "dh-7f3a/container/"+container) + require.Equal(t, registry.sourceResourceID(SourceAgent, "host-tower"), hostAID, "the Docker host merges into its agent's machine") + require.NotEqual(t, hostAID, hostBID) + require.NotEmpty(t, containerID) + + bound := map[string]string{ + "docker:dh-91c0": hostBID, + "docker:dh-91c0/service/" + serviceID: serviceResourceID, + "docker:dh-7f3a": hostAID, + "docker:dh-7f3a/service/" + serviceID: serviceResourceID, + } + unbound := []string{ + "docker:tower", // hostname + "docker:dh-7f3a/service/web", // service name + "docker:dh-c4d2/service/" + otherServiceID, // its cluster has a service without an ID + "docker-service:web", // hostless service name + "docker:dh-7f3a/worker", // container name + "docker:dh-7f3a/" + container[:12], // shortened container ID + "docker:unknown", + } + for i, ref := range append([]string{"docker:dh-7f3a", "docker:dh-7f3a/service/" + serviceID}, unbound...) { + if ref == "docker:dh-7f3a/worker" { + continue // journaled above + } + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "fired-" + ref, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Minute + time.Duration(i)*time.Second)})) + } + requireHistoryReferenceBindings(t, store, bound, unbound) + // Names and shortened IDs are never retried against later inventory. + require.Contains(t, adapter.legacyHistory.pending, "docker:tower") + for _, ref := range []string{"docker:dh-7f3a/worker", "docker:dh-7f3a/" + container[:12], "docker-service:web"} { + require.NotContains(t, adapter.legacyHistory.pending, ref) + } + got, err := store.GetRecentChangesFiltered(containerID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + require.Empty(t, got, "a container name or short ID never joins the container's history") + // While its cluster has a service without an ID, a service reference is a + // conflict and must not follow a binding recorded earlier. + collision := "docker:dh-c4d2/service/" + otherServiceID + apiID := historyIdentityResourceID(t, registry, ResourceTypeDockerService, "api") + require.NoError(t, store.RecordChangeWithSourceIdentity(ResourceChange{ID: "earlier-api", ResourceID: apiID, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now}, collision)) + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "conflict-api", ResourceID: collision, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(2 * time.Minute)})) + // The same holds once the service with that ID has left inventory. + hostC.Services = hostC.Services[1:] + adapter.PopulateFromSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ID: "host-tower", Hostname: "tower", MachineID: "0123456789abcdef", Status: "online", LastSeen: now}}, + DockerHosts: []models.DockerHost{hostA, hostB, hostC}, + }) + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "conflict-api-removed", ResourceID: collision, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(3 * time.Minute)})) + for _, id := range []string{"conflict-api", "conflict-api-removed"} { + var recordedAs string + require.NoError(t, store.db.QueryRow(`SELECT canonical_id FROM resource_changes WHERE id = ?`, id).Scan(&recordedAs)) + require.Equal(t, collision, recordedAs, id) + } + + // After the service leaves inventory its alerts follow the retained binding; + // a container name still binds nothing. + removed := NewMonitorAdapter(NewRegistry(store)) + for _, ref := range []string{"docker:dh-91c0/service/" + serviceID, "docker:dh-7f3a/worker"} { + require.NoError(t, removed.RecordChange(ResourceChange{ID: "resolved-" + ref, ResourceID: ref, Kind: ChangeAlertResolved, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Hour)})) + } + got, err = store.GetRecentChangesFiltered(serviceResourceID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired, ChangeAlertResolved}}) + require.NoError(t, err) + require.Len(t, got, 3) + require.Equal(t, "resolved-docker:dh-91c0/service/"+serviceID, got[0].ID) + require.Equal(t, serviceResourceID, got[0].ResourceID) + got, err = store.GetRecentChanges("docker:dh-7f3a/worker", time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 2) + require.Equal(t, "docker:dh-7f3a/worker", got[0].ResourceID) +} + +// Sub-resource alerts (ZFS pools and devices, host filesystems, disks, RAID +// arrays, sensors, the Unraid array) name no resource of their own. They join +// the history of the owner their reference carries a durable ID for: the +// storage, the host, or the Unraid array storage. Device labels never bind a +// physical disk, and an owner of the wrong type or a name binds nothing. +func TestHistoryIdentityMonitorAdapterResolvesSubResourceReferences(t *testing.T) { + store, err := NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + now := time.Now().UTC().Truncate(time.Second) + snapshot := proxmoxHistoryIdentitySnapshot(now) + snapshot.Storage = []models.Storage{{ID: "lab-pve1-local-zfs", Name: "local-zfs", Node: "pve1", Instance: "lab", Type: "zfspool", Status: "available", Total: 100, Used: 10, LastSeen: now}} + snapshot.Hosts[0].Sensors.SMART = []models.HostDiskSMART{{Device: "sda", Serial: "S3Z1NB0K", Model: "Samsung SSD", Health: "PASSED", Temperature: 34}} + snapshot.Hosts = append(snapshot.Hosts, models.Host{ID: "host-nas", Hostname: "nas", MachineID: "fedcba9876543210", Status: "online", LastSeen: now, + Unraid: &models.HostUnraidStorage{ArrayStarted: true, ArrayState: "STARTED", Disks: []models.HostUnraidDisk{{Name: "disk1", Device: "sdb", Role: "data", Status: "DISK_OK", Serial: "WD-1"}}}}) + adapter := NewMonitorAdapter(NewRegistry(store)) + adapter.PopulateFromSnapshot(snapshot) + registry := adapter.currentRegistry() + nodeID := historyIdentityResourceID(t, registry, ResourceTypeAgent, "pve1") + storageID := registry.sourceResourceID(SourceProxmox, "lab-pve1-local-zfs") + unraidID := registry.sourceResourceID(SourceAgent, "host-nas/storage:unraid-array") + diskID := registry.sourceResourceID(SourceAgent, HostSMARTDiskSourceID(snapshot.Hosts[0], snapshot.Hosts[0].Sensors.SMART[0])) + require.NotEmpty(t, storageID) + require.NotEmpty(t, unraidID) + require.NotEmpty(t, diskID) + + bound := map[string]string{ + "lab-pve1-local-zfs/zfs-pool:local-zfs": storageID, + "lab-pve1-local-zfs/zfs-pool:local-zfs/device:sda2": storageID, + "agent:host-pve1/disk:var": nodeID, + "agent:host-pve1/disk:sda": nodeID, + "agent:host-pve1/disk_temp:sda": nodeID, + "agent:host-pve1/raid:md0": nodeID, + "agent:host-pve1/custom:fan1": nodeID, + "agent:host-nas/storage:unraid-array": unraidID, + } + unbound := []string{ + "lab-pve1/zfs-pool:local-zfs", // the owner part names a node, not a storage + "local-zfs/zfs-pool:local-zfs", // a storage name, not its ID + "agent:pve1/disk:var", // a hostname, not the host ID + "agent:host-pve1/service:sshd", // not a sub-resource shape any producer emits + "agent:host-pve1/disk:", + } + refs := append([]string{}, unbound...) + for ref := range bound { + refs = append(refs, ref) + } + for i, ref := range refs { + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "fired-" + ref, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Duration(i) * time.Second)})) + } + requireHistoryReferenceBindings(t, store, bound, unbound) + got, err := store.GetRecentChangesFiltered(diskID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + require.Empty(t, got, "a kernel device label never binds a physical disk") + + // Recovery after the storage left inventory follows the retained binding. + pool := "lab-pve1-local-zfs/zfs-pool:local-zfs" + removed := NewMonitorAdapter(NewRegistry(store)) + require.NoError(t, removed.RecordChange(ResourceChange{ID: "resolved-" + pool, ResourceID: pool, Kind: ChangeAlertResolved, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Hour)})) + got, err = store.GetRecentChangesFiltered(storageID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertResolved}}) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, storageID, got[0].ResourceID) + + // An owner reference that two resources answer to, or that names a resource + // of another type, binds nothing and must not follow the retained binding. + vmID := historyIdentityResourceID(t, registry, ResourceTypeVM, "web") + for _, conflict := range []struct { + name string + source DataSource + }{{"ambiguous", SourcePBS}, {"wrong-type", SourceProxmox}} { + registry.mu.Lock() + delete(registry.bySource[SourcePBS], "lab-pve1-local-zfs") + if registry.bySource[conflict.source] == nil { + registry.bySource[conflict.source] = make(map[string]string) + } + registry.bySource[conflict.source]["lab-pve1-local-zfs"] = vmID + registry.mu.Unlock() + id := conflict.name + "-" + pool + require.NoError(t, adapter.RecordChange(ResourceChange{ID: id, ResourceID: pool, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(2 * time.Hour)})) + var recordedAs string + require.NoError(t, store.db.QueryRow(`SELECT canonical_id FROM resource_changes WHERE id = ?`, id).Scan(&recordedAs)) + require.Equal(t, pool, recordedAs, conflict.name) + bound, _, err := store.ResolveHistorySourceIdentity(pool) + require.NoError(t, err) + require.Equal(t, storageID, bound, conflict.name) + } + // A conflicting reference without a binding is retried, like one found at + // startup, and binds once a generation names its owner unambiguously. + device := "lab-pve1-local-zfs/zfs-pool:local-zfs/device:sdc2" + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "fired-" + device, ResourceID: device, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(3 * time.Hour)})) + require.Contains(t, adapter.legacyHistory.pending, device) + adapter.PopulateFromSnapshot(snapshot) + deviceBinding, found, err := store.ResolveHistorySourceIdentity(device) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, storageID, deviceBinding) +} diff --git a/internal/unifiedresources/monitor_adapter.go b/internal/unifiedresources/monitor_adapter.go index 94795237b4..26a39e0958 100644 --- a/internal/unifiedresources/monitor_adapter.go +++ b/internal/unifiedresources/monitor_adapter.go @@ -198,7 +198,8 @@ func (a *MonitorAdapter) currentRegistry() *ResourceRegistry { // RecordChange forwards canonical resource-history events into the underlying // resource store when monitoring has a durable store attached. Alert lifecycle // events carry source-native references (Proxmox node and guest IDs, agent -// IDs); they resolve here so history reads by canonical ID include them. +// IDs, Docker host and Swarm service IDs, sub-resource references); they +// resolve here so history reads by canonical ID include them. func (a *MonitorAdapter) RecordChange(change ResourceChange) error { registry := a.currentRegistry() if registry == nil || registry.store == nil { @@ -229,16 +230,19 @@ func (a *MonitorAdapter) RecordChange(change ResourceChange) error { } } else if resolvedID, claimed := registry.resolveHistoryReference(sourceRef); resolvedID != "" { change.ResourceID = resolvedID - } else if hasHistory && !claimed && !isDockerHistoryReference(sourceRef) { - // No resource answers to it: keep the binding an earlier event recorded. - id, found, err := history.ResolveHistorySourceIdentity(CanonicalResourceID(sourceRef)) - if err != nil { - return err - } - if found { - change.ResourceID = id - } else { - unbound = true + } else if hasHistory && !isDockerNameHistoryReference(sourceRef) { + // A later generation may name the resource or settle a conflict. + unbound = true + if !claimed { + // No resource answers to it: keep the binding an earlier event recorded. + id, found, err := history.ResolveHistorySourceIdentity(CanonicalResourceID(sourceRef)) + if err != nil { + return err + } + if found { + change.ResourceID = id + unbound = false + } } } if hasHistory && CanonicalResourceID(sourceRef) != CanonicalResourceID(change.ResourceID) {