diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 2384d6c7c4..ced11b1f90 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -17,6 +17,24 @@ ## Purpose +### Automatic system-mount exclusions respect directory boundaries + +The shared filesystem filter excludes a named system directory and its +descendants, not independent local mounts with the same text prefix. Volumes +such as `/snapshots`, `/runtime` and `/var/lib/docker-data` remain eligible for +capacity, current usage and ordinary alert evaluation without an explicit +include override. Genuine system directories, virtual/read-only/remote types +and container layers remain excluded. Explicit operator wildcard/exact excludes +and include precedence are unchanged; guest commands and their safety admission +are untouched. + +`mount_boundary_test.go` in `pkg/fsfilters` and `internal/hostmetrics` cover the +directory boundary, real pre-usage collector admission and skip/override +controls. `filesystem_mount_boundary_test.go` carries local mount counters +through QGA aggregation and both report/state/canonical-read paths, and host +usage alert onset/recovery. These synthetic controls do not establish native +guest polling, installed agent recovery or completion of a reported incident. + ### QEMU Windows capacity is owned by the volume, not the physical drive The guest filesystem aggregate uses the reported Windows volume GUID when diff --git a/internal/hostmetrics/mount_boundary_test.go b/internal/hostmetrics/mount_boundary_test.go new file mode 100644 index 0000000000..44ec3c2d29 --- /dev/null +++ b/internal/hostmetrics/mount_boundary_test.go @@ -0,0 +1,50 @@ +package hostmetrics + +import ( + "context" + "reflect" + "testing" + + godisk "github.com/shirou/gopsutil/v4/disk" +) + +func TestCollectDisksKeepsSystemNamedSiblingVolumes(t *testing.T) { + origPartitions, origUsage := diskPartitions, diskUsage + t.Cleanup(func() { diskPartitions, diskUsage = origPartitions, origUsage }) + diskPartitions = func(context.Context, bool) ([]godisk.PartitionStat, error) { + return []godisk.PartitionStat{ + {Device: "/dev/vda1", Mountpoint: "/snapshots", Fstype: "ext4"}, + {Device: "/dev/vdb1", Mountpoint: "/runtime", Fstype: "xfs"}, + {Device: "/dev/vdc1", Mountpoint: "/var/lib/docker-data", Fstype: "ext4"}, + {Device: "snap-image", Mountpoint: "/snap/image", Fstype: "squashfs"}, + {Device: "runtime", Mountpoint: "/run/worker", Fstype: "ext4"}, + {Device: "remote", Mountpoint: "/snapshots-remote", Fstype: "nfs4"}, + {Device: "/dev/vdd1", Mountpoint: "/snapshots-private", Fstype: "ext4"}, + }, nil + } + var calls []string + diskUsage = func(_ context.Context, path string) (*godisk.UsageStat, error) { + calls = append(calls, path) + return &godisk.UsageStat{Total: 1000, Used: 950, Free: 50, UsedPercent: 95}, nil + } + disks := collectDisks(context.Background(), []string{"/snapshots-private"}) + want := []string{"/snapshots", "/runtime", "/var/lib/docker-data"} + if !reflect.DeepEqual(calls, want) { + t.Fatalf("usage probes = %v, want only eligible local volumes %v", calls, want) + } + if len(disks) != len(want) { + t.Fatalf("local volumes missing from agent inventory: %+v", disks) + } + seen := map[string]bool{} + for _, disk := range disks { + seen[disk.Mountpoint] = true + if disk.TotalBytes != 1000 || disk.UsedBytes != 950 || disk.FreeBytes != 50 || disk.Usage != 95 || disk.ExplicitlyIncluded { + t.Fatalf("automatically admitted volume lost usage or became an override: %+v", disk) + } + } + for _, mount := range want { + if !seen[mount] { + t.Errorf("missing local volume %q", mount) + } + } +} diff --git a/internal/monitoring/filesystem_mount_boundary_test.go b/internal/monitoring/filesystem_mount_boundary_test.go new file mode 100644 index 0000000000..34fbc175bc --- /dev/null +++ b/internal/monitoring/filesystem_mount_boundary_test.go @@ -0,0 +1,117 @@ +package monitoring + +import ( + "encoding/json" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" + agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" +) + +func TestVMFilesystemKeepsSystemNamedSiblingVolumes(t *testing.T) { + paths := []string{"/snapshots", "/runtime", "/var/lib/docker-data", "/snap/image", "/run/worker", "/var/lib/docker/overlay2"} + var filesystems []proxmox.VMFileSystem + for i, mount := range paths { + filesystems = append(filesystems, proxmox.VMFileSystem{ + Mountpoint: mount, Type: "ext4", Disk: mount, + TotalBytes: 1000, UsedBytes: uint64(950 - i*100), + }) + } + summary := (&Monitor{}).summarizeVMFSInfo("fixture", proxmox.ClusterResource{}, filesystems) + if summary.invalidBytes || summary.totalBytes != 3000 || summary.usedBytes != 2550 || len(summary.individualDisks) != 3 { + t.Fatalf("independent local volumes missing from QGA capacity: %+v", summary) + } + for i, disk := range summary.individualDisks { + if disk.Mountpoint != paths[i] || disk.Device != paths[i] || disk.Total != 1000 || disk.Used != int64(950-i*100) { + t.Fatalf("volume metadata/counters changed: %+v", disk) + } + } +} + +func TestAgentReportsKeepSystemNamedSiblingVolumes(t *testing.T) { + for _, kind := range []string{"host", "docker"} { + t.Run(kind, func(t *testing.T) { + m := newTestMonitor(t) + m.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + cfg := m.alertManager.GetConfig() + cfg.Enabled, cfg.ActivationState = true, alerts.ActivationActive + cfg.AgentDefaults = alerts.ThresholdConfig{Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 80}} + cfg.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0}} + cfg.TimeThresholds = map[string]int{"all": 0} + cfg.MinimumDelta, cfg.SuppressionWindow = 0, 0 + m.alertManager.UpdateConfig(cfg) + var disks []agentshost.Disk + if err := json.Unmarshal([]byte(`[ + {"device":"/dev/vda1","mountpoint":"/snapshots","type":"ext4","totalBytes":1000,"usedBytes":950,"freeBytes":50,"usage":95}, + {"device":"/dev/vdb1","mountpoint":"/runtime","type":"xfs","totalBytes":1000,"usedBytes":400,"freeBytes":600,"usage":40}, + {"device":"/dev/vdc1","mountpoint":"/var/lib/docker-data","type":"ext4","totalBytes":1000,"usedBytes":700,"freeBytes":300,"usage":70}, + {"device":"image","mountpoint":"/snap/image","type":"squashfs","totalBytes":1000,"usedBytes":1000,"usage":100}, + {"device":"runtime","mountpoint":"/run/worker","type":"ext4","totalBytes":1000,"usedBytes":1000,"usage":100}, + {"device":"remote","mountpoint":"/snapshots-remote","type":"nfs4","totalBytes":1000,"usedBytes":1000,"usage":100} + ]`), &disks); err != nil { + t.Fatal(err) + } + for phase := 0; phase < 2; phase++ { + if phase == 1 { + for i := range disks { + disks[i].UsedBytes, disks[i].FreeBytes, disks[i].Usage = 100, 900, 10 + } + } + now := time.Now().UTC() + var native []models.Disk + var canonical []unifiedresources.DiskInfo + var stored []models.Disk + if kind == "host" { + host, err := m.ApplyHostReport(agentshost.Report{ + Agent: agentshost.AgentInfo{ID: "boundary-host", IntervalSeconds: 30}, + Host: agentshost.HostInfo{ID: "boundary-machine", Hostname: "boundary-host"}, + Disks: disks, Timestamp: now, + }, nil) + if err != nil { + t.Fatal(err) + } + native = host.Disks + stored = m.state.GetSnapshot().Hosts[0].Disks + canonical = m.GetUnifiedReadState().Hosts()[0].Disks() + // Actual report admission evaluates the restored near-full + // volume, then clears its alert on the next healthy report. + active := m.alertManager.GetActiveAlerts() + if phase == 0 { + if len(active) != 1 || active[0].Metadata["mountpoint"] != "/snapshots" || active[0].Value != 95 { + t.Fatalf("near-full local volume did not reach normal alert evaluation: %+v", active) + } + } else if len(active) != 0 { + t.Fatalf("healthy volume retained an active usage alert: %+v", active) + } + } else { + host, err := m.ApplyDockerReport(agentsdocker.Report{ + Agent: agentsdocker.AgentInfo{ID: "boundary-docker", IntervalSeconds: 30}, + Host: agentsdocker.HostInfo{MachineID: "boundary-docker-machine", Hostname: "boundary-docker", Disks: disks}, + Timestamp: now, + }, nil) + if err != nil { + t.Fatal(err) + } + native = host.Disks + stored = m.state.GetSnapshot().DockerHosts[0].Disks + canonical = m.GetUnifiedReadState().DockerHosts()[0].Disks() + } + if len(native) != 3 || len(stored) != 3 || len(canonical) != 3 { + t.Fatalf("local volumes missing at report/state/read boundaries: %v / %v / %v", native, stored, canonical) + } + for i := 0; i < 3; i++ { + want := disks[i] + if native[i].Mountpoint != want.Mountpoint || native[i].Total != want.TotalBytes || native[i].Used != want.UsedBytes || native[i].Usage != want.Usage || + stored[i] != native[i] || canonical[i].Mountpoint != want.Mountpoint || canonical[i].Total != want.TotalBytes || canonical[i].Used != want.UsedBytes || canonical[i].Usage != want.Usage { + t.Fatalf("phase %d lost current local volume identity/usage: %+v / %+v / %+v", phase, native[i], stored[i], canonical[i]) + } + } + } + }) + } +} diff --git a/pkg/fsfilters/filters.go b/pkg/fsfilters/filters.go index 41f145236d..842c0cf628 100644 --- a/pkg/fsfilters/filters.go +++ b/pkg/fsfilters/filters.go @@ -164,7 +164,10 @@ func ShouldSkipFilesystem(fsType, mountpoint string, totalBytes, usedBytes uint6 // Check special mountpoint prefixes for _, prefix := range specialMountPrefixes { - if strings.HasPrefix(mountpoint, prefix) { + // These identify directories, not arbitrary string prefixes. A local + // /snapshots or /runtime volume is not a child of /snap or /run. + directory := strings.TrimSuffix(prefix, "/") + if mountpoint == directory || strings.HasPrefix(mountpoint, directory+"/") { reasons = append(reasons, "special-mountpoint") break } diff --git a/pkg/fsfilters/mount_boundary_test.go b/pkg/fsfilters/mount_boundary_test.go new file mode 100644 index 0000000000..5de61899a6 --- /dev/null +++ b/pkg/fsfilters/mount_boundary_test.go @@ -0,0 +1,43 @@ +package fsfilters + +import ( + "strings" + "testing" +) + +func TestSystemMountFiltersRespectDirectoryBoundaries(t *testing.T) { + // Include the roots as well as descendants. Paths that merely share a + // directory's spelling must remain eligible, even when they are full. + for _, directory := range []string{ + "/dev", "/proc", "/sys", "/run", "/var/run", "/var/lib/docker", + "/var/lib/containers", "/snap", "/System/Volumes", + "/Library/Developer/CoreSimulator/Volumes", + } { + t.Run(directory, func(t *testing.T) { + for _, path := range []string{directory, directory + "/", directory + "/child/mount"} { + skip, reasons := ShouldSkipFilesystem("ext4", path, 1000, 1000) + if !skip || !strings.Contains(strings.Join(reasons, ","), "special-mountpoint") { + t.Errorf("system directory %q escaped the filter: %v", path, reasons) + } + } + for _, suffix := range []string{"s", "-data", ".backup"} { + for _, path := range []string{directory + suffix, directory + suffix + "/child/mount"} { + if skip, reasons := ShouldSkipFilesystem("ext4", path, 1000, 1000); skip { + t.Errorf("independent full volume %q was hidden: %v", path, reasons) + } + } + } + }) + } + for _, fsType := range []string{"tmpfs", "squashfs", "nfs4", "cifs", "overlay"} { + if skip, _ := ShouldSkipFilesystem(fsType, "/snapshots", 1000, 1000); !skip { + t.Errorf("directory-boundary repair bypassed %q type filtering", fsType) + } + } + // Operator patterns deliberately support textual prefixes; this repair + // must not narrow a user's explicit wildcard or turn an exact path broad. + if !MatchesDiskExclude("/dev/vdb", "/snapshots", []string{"/snap*"}) || + MatchesDiskExclude("/dev/vdb", "/snapshots", []string{"/snap"}) { + t.Fatal("explicit operator pattern semantics changed") + } +}