From c8cead7d5ac637ee9c91f538af0af8172ce734ac Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:34:11 +0100 Subject: [PATCH] fix(agents): preserve explicit disk includes at ingestion Carry the operator include override in disk reports so server filtering does not discard selected tmpfs mounts again. Keep automatic filtering for unmarked reports and agent exclusion precedence. Cover both ingest paths, collection, forwarding and wire compatibility for #1875. Change-source: pulse-maintainer --- .../v6/internal/subsystems/agent-lifecycle.md | 17 +++++++ .../v6/internal/subsystems/monitoring.md | 18 +++++++ internal/hostagent/agent_metrics_test.go | 8 +++- internal/hostmetrics/collector.go | 17 +++---- internal/hostmetrics/collector_test.go | 2 +- internal/monitoring/monitor_agents.go | 9 ++-- .../monitoring/monitor_host_agents_test.go | 48 +++++++++++++++++++ pkg/agents/host/report.go | 21 ++++---- pkg/agents/host/report_test.go | 24 ++++++++++ 9 files changed, 140 insertions(+), 24 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 45f42419d..95516b861 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -15,6 +15,23 @@ ## Purpose +### Explicit filesystem selections survive the report boundary + +Disk reports carry optional `explicitlyIncluded` evidence when the collector +matches the operator's disk-include setting. Explicit exclusions still win +before usage collection; inclusion does not fabricate capacity when usage fails +or reports zero total. Host and Docker snapshot copies preserve this field. +It changes metric selection only, not enrollment, token scopes, identity or +command authority. + +The flag is omitted when false. Older agents therefore retain default server +filtering; older servers ignore the additive field. Preserving selected tmpfs +through ingestion requires both agent and server support, not a server-only +upgrade. `TestDiskExplicitIncludeWireCompatibility` pins omission and round +trip; `TestIssue1875MountinfoToExplicitDiskCollection` pins include/exclude +selection and marker production; `TestBuildReportForwardsExplicitDiskIncludesAndExcludes` +pins report forwarding. These are synthetic proofs, not estate acceptance. + Historical incident reads preserve canonical event targets and treat resource aliases as read selectors. They do not grant command authority to an alias or infer present agent liveness from an old alert closure. A failed canonical read diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 841f35a0e..1514be7b2 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -17,6 +17,24 @@ ## Purpose +### Preserve explicit agent filesystem selection at ingestion + +Both `ApplyHostReport` and `ApplyDockerReport` honour a disk's optional +`explicitlyIncluded` marker when applying automatic filesystem filtering. +Selected tmpfs mounts must not be discarded a second time after the agent +has admitted them. Unmarked virtual/system filesystems remain filtered and +ordinary physical filesystem handling is unchanged. Selection evidence does +not confer command authority or imply successful observation beyond the +reported metrics. + +`TestAgentReportsPreserveExplicitDiskIncludes` in +`internal/monitoring/monitor_host_agents_test.go` exercises JSON-decoded disks +through both ingestion paths: two selected equal-capacity tmpfs mounts survive, +unselected /run is rejected, and root remains. Agent-side exclusions still +precede marker production. Legacy reports without the marker retain the +existing default policy; matched agent and server support is required. +This proves ingestion behaviour, not reporter installation or release delivery. + ### TrueNAS persistent-session liveness and successful poll cadence Authenticated JSON-RPC WebSocket sessions send transport-only PING controls diff --git a/internal/hostagent/agent_metrics_test.go b/internal/hostagent/agent_metrics_test.go index 854aa0b15..9c70c0ba1 100644 --- a/internal/hostagent/agent_metrics_test.go +++ b/internal/hostagent/agent_metrics_test.go @@ -491,7 +491,7 @@ func TestBuildReportForwardsExplicitDiskIncludesAndExcludes(t *testing.T) { metricsWithDiskFiltersFn: func(_ context.Context, exclude, include []string) (hostmetrics.Snapshot, error) { gotExclude = append([]string(nil), exclude...) gotInclude = append([]string(nil), include...) - return hostmetrics.Snapshot{}, nil + return hostmetrics.Snapshot{Disks: []agentshost.Disk{{Mountpoint: "/mnt/containers", Type: "tmpfs", TotalBytes: 1024, ExplicitlyIncluded: true}}}, nil }, } agent, err := New(Config{ @@ -506,9 +506,13 @@ func TestBuildReportForwardsExplicitDiskIncludesAndExcludes(t *testing.T) { t.Fatalf("New() failed: %v", err) } - if _, err := agent.buildReport(context.Background()); err != nil { + report, err := agent.buildReport(context.Background()) + if err != nil { t.Fatalf("buildReport() failed: %v", err) } + if len(report.Disks) != 1 || !report.Disks[0].ExplicitlyIncluded || report.Disks[0].Mountpoint != "/mnt/containers" { + t.Fatalf("explicit disk lost in report: %+v", report.Disks) + } if got := strings.Join(gotExclude, ","); got != "/mnt/private" { t.Fatalf("disk excludes = %q, want /mnt/private", got) } diff --git a/internal/hostmetrics/collector.go b/internal/hostmetrics/collector.go index ad05dfff9..072be6108 100644 --- a/internal/hostmetrics/collector.go +++ b/internal/hostmetrics/collector.go @@ -474,14 +474,15 @@ func collectDisksWithIncludes(ctx context.Context, diskExclude, diskInclude []st } disks = append(disks, agentshost.Disk{ - Device: part.Device, - Mountpoint: part.Mountpoint, - Filesystem: part.Fstype, - Type: part.Fstype, - TotalBytes: int64(usage.Total), - UsedBytes: int64(usage.Used), - FreeBytes: int64(usage.Free), - Usage: usage.UsedPercent, + ExplicitlyIncluded: explicitlyIncluded, + Device: part.Device, + Mountpoint: part.Mountpoint, + Filesystem: part.Fstype, + Type: part.Fstype, + TotalBytes: int64(usage.Total), + UsedBytes: int64(usage.Used), + FreeBytes: int64(usage.Free), + Usage: usage.UsedPercent, }) } diff --git a/internal/hostmetrics/collector_test.go b/internal/hostmetrics/collector_test.go index 9ed8c843d..5fb4b39e5 100644 --- a/internal/hostmetrics/collector_test.go +++ b/internal/hostmetrics/collector_test.go @@ -405,7 +405,7 @@ func TestIssue1875MountinfoToExplicitDiskCollection(t *testing.T) { var got []string for _, disk := range disks { got = append(got, disk.Mountpoint) - if disk.Filesystem != "tmpfs" || disk.TotalBytes != 1024 || disk.Usage != 75 { + if !disk.ExplicitlyIncluded || disk.Filesystem != "tmpfs" || disk.TotalBytes != 1024 || disk.Usage != 75 { t.Fatalf("incorrect capacity/type: %+v", disk) } } diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go index d455a8976..a8acef02b 100644 --- a/internal/monitoring/monitor_agents.go +++ b/internal/monitoring/monitor_agents.go @@ -2449,8 +2449,8 @@ func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *con disks := make([]models.Disk, 0, len(report.Host.Disks)) for _, disk := range report.Host.Disks { // Filter virtual/system filesystems (same as ApplyHostReport) to avoid - // inflated disk totals from tmpfs, overlayfs, etc. - if shouldSkip, _ := fsfilters.ShouldSkipFilesystem(disk.Type, disk.Mountpoint, uint64(disk.TotalBytes), uint64(disk.UsedBytes)); shouldSkip { + // inflated disk totals from tmpfs, overlayfs, etc., unless explicitly selected. + if shouldSkip, _ := fsfilters.ShouldSkipFilesystem(disk.Type, disk.Mountpoint, uint64(disk.TotalBytes), uint64(disk.UsedBytes)); shouldSkip && !disk.ExplicitlyIncluded { continue } disks = append(disks, models.Disk{ @@ -3287,8 +3287,9 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config. for _, disk := range report.Disks { // Filter virtual/system filesystems and read-only filesystems to avoid cluttering // the UI with tmpfs, devtmpfs, /dev, /run, /sys, docker overlay mounts, snap mounts, - // immutable OS images, etc. (issues #505, #690, #790). - if shouldSkip, _ := fsfilters.ShouldSkipFilesystem(disk.Type, disk.Mountpoint, uint64(disk.TotalBytes), uint64(disk.UsedBytes)); shouldSkip { + // immutable OS images, etc. (issues #505, #690, #790). Preserve the + // agent operator's explicit include override rather than filtering it again. + if shouldSkip, _ := fsfilters.ShouldSkipFilesystem(disk.Type, disk.Mountpoint, uint64(disk.TotalBytes), uint64(disk.UsedBytes)); shouldSkip && !disk.ExplicitlyIncluded { continue } diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go index 164796133..f8d32ea0e 100644 --- a/internal/monitoring/monitor_host_agents_test.go +++ b/internal/monitoring/monitor_host_agents_test.go @@ -5836,3 +5836,51 @@ func TestApplyHostReportBridgeIdentity(t *testing.T) { }) } } + +// Explicit includes must survive the wire and both server report paths. +func TestAgentReportsPreserveExplicitDiskIncludes(t *testing.T) { + var disks []agentshost.Disk + if err := json.Unmarshal([]byte(`[ + {"device":"log2ram","mountpoint":"/var/log","type":"tmpfs","totalBytes":1024,"usedBytes":768,"explicitlyIncluded":true}, + {"device":"tmpfs","mountpoint":"/mnt/ramdisk/plex-transcode","type":"tmpfs","totalBytes":1024,"usedBytes":256,"explicitlyIncluded":true}, + {"device":"tmpfs","mountpoint":"/run","type":"tmpfs","totalBytes":1024}, + {"device":"/dev/sda","mountpoint":"/","type":"ext4","totalBytes":4096,"usedBytes":1024} + ]`), &disks); err != nil { + t.Fatal(err) + } + check := func(t *testing.T, got []models.Disk) { + t.Helper() + if len(got) != 3 { + t.Fatalf("accepted disks = %+v, want both selected tmpfs and root only", got) + } + for i, want := range []string{"/var/log", "/mnt/ramdisk/plex-transcode", "/"} { + if got[i].Mountpoint != want || got[i].Total != []int64{1024, 1024, 4096}[i] { + t.Fatalf("disk %d = %+v", i, got[i]) + } + } + } + t.Run("host", func(t *testing.T) { + m := newTestMonitor(t) + host, err := m.ApplyHostReport(agentshost.Report{ + Agent: agentshost.AgentInfo{ID: "include-host", IntervalSeconds: 30}, + Host: agentshost.HostInfo{ID: "include-machine", Hostname: "include-host"}, + Disks: disks, Timestamp: time.Now().UTC(), + }, nil) + if err != nil { + t.Fatal(err) + } + check(t, host.Disks) + }) + t.Run("docker", func(t *testing.T) { + m := newTestMonitor(t) + host, err := m.ApplyDockerReport(agentsdocker.Report{ + Agent: agentsdocker.AgentInfo{ID: "include-docker", IntervalSeconds: 30}, + Host: agentsdocker.HostInfo{MachineID: "include-docker-machine", Hostname: "include-docker", Disks: disks}, + Timestamp: time.Now().UTC(), + }, nil) + if err != nil { + t.Fatal(err) + } + check(t, host.Disks) + }) +} diff --git a/pkg/agents/host/report.go b/pkg/agents/host/report.go index d484c4733..8e27177aa 100644 --- a/pkg/agents/host/report.go +++ b/pkg/agents/host/report.go @@ -311,15 +311,18 @@ type MemoryMetric struct { // Disk represents disk utilisation metrics. type Disk struct { - Device string `json:"device,omitempty"` - Mountpoint string `json:"mountpoint,omitempty"` - Filesystem string `json:"filesystem,omitempty"` - Type string `json:"type,omitempty"` - TotalBytes int64 `json:"totalBytes,omitempty"` - UsedBytes int64 `json:"usedBytes,omitempty"` - FreeBytes int64 `json:"freeBytes,omitempty"` - Usage float64 `json:"usage,omitempty"` - ZFSDatasets []ZFSDataset `json:"zfsDatasets,omitempty"` + // ExplicitlyIncluded carries the operator disk-include override through server filtering. + // Absent on older agents, which retain the default filesystem policy. + ExplicitlyIncluded bool `json:"explicitlyIncluded,omitempty"` + Device string `json:"device,omitempty"` + Mountpoint string `json:"mountpoint,omitempty"` + Filesystem string `json:"filesystem,omitempty"` + Type string `json:"type,omitempty"` + TotalBytes int64 `json:"totalBytes,omitempty"` + UsedBytes int64 `json:"usedBytes,omitempty"` + FreeBytes int64 `json:"freeBytes,omitempty"` + Usage float64 `json:"usage,omitempty"` + ZFSDatasets []ZFSDataset `json:"zfsDatasets,omitempty"` } // ZFSDataset is a bounded, read-only projection of `zfs list` for one pool. diff --git a/pkg/agents/host/report_test.go b/pkg/agents/host/report_test.go index df78fa523..4d085356e 100644 --- a/pkg/agents/host/report_test.go +++ b/pkg/agents/host/report_test.go @@ -850,3 +850,27 @@ func TestReportProxmoxLXCInventoryJSONRoundTrip(t *testing.T) { t.Fatalf("nil Proxmox LXC inventory should be omitted: %s", bare) } } + +func TestDiskExplicitIncludeWireCompatibility(t *testing.T) { + for _, included := range []bool{false, true} { + disk := Disk{Mountpoint: "/var/log", Type: "tmpfs", TotalBytes: 1024, ExplicitlyIncluded: included} + data, err := json.Marshal(disk) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + t.Fatal(err) + } + if _, present := fields["explicitlyIncluded"]; present != included { + t.Fatalf("include field presence = %v, want %v: %s", present, included, data) + } + var decoded Disk + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + if decoded.ExplicitlyIncluded != included || decoded.Mountpoint != disk.Mountpoint || decoded.TotalBytes != disk.TotalBytes || decoded.Type != disk.Type { + t.Fatalf("round trip = %+v, want %+v", decoded, disk) + } + } +}