Keep offline mock hosts on the host alert lifecycle

This commit is contained in:
Pulse Test
2026-08-28 00:07:17 +01:00
parent ab92fdbc35
commit 413288d049
3 changed files with 74 additions and 1 deletions
@@ -456,6 +456,13 @@ on this path: agent ingest is not gated on `PULSE_MOCK_KEEP_REAL_POLLING`, and
the read state is mock-substituted either way, so injecting real hosts would
only graft them onto fixture data.
Mock alert evaluation preserves the live Docker connectivity boundary. An
explicitly offline Docker fixture routes through `HandleDockerHostOffline`, not
the fresh-report `CheckDockerHost` path. Its last container states are unknown
supporting inventory rather than a new batch of independent exits, so the
confirmed host incident clears child alerts instead of producing one alert per
container.
Host and container-runtime disk collection supports an explicit include list
for filesystems hidden by Pulse's automatic virtual/container filtering. The
include list is bounded to that automatic filter; explicit disk exclusions
+15 -1
View File
@@ -11,6 +11,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
@@ -629,10 +630,23 @@ func (m *Monitor) checkMockAlerts() {
// against mock data.
log.Debug().Int("dockerHostCount", len(state.DockerHosts)).Msg("checking docker alerts")
for _, dockerHost := range state.DockerHosts {
m.alertManager.CheckDockerHost(dockerHost)
m.checkMockDockerHostAlerts(dockerHost)
}
// Cache the latest alert snapshots directly in the mock data so the API can serve
// mock state without needing to grab the alert manager lock again.
mock.UpdateAlertSnapshots(m.alertManager.GetActiveAlerts(), m.alertManager.GetRecentlyResolved())
}
// checkMockDockerHostAlerts preserves the same evidence boundary as live
// agent monitoring. An explicitly offline fixture is missing fresh container
// telemetry; its last container states must not be reinterpreted as a fresh
// batch of independent exits. The host connectivity lifecycle owns that
// outage and clears child alerts once the offline confirmation floor is met.
func (m *Monitor) checkMockDockerHostAlerts(host models.DockerHost) {
if strings.EqualFold(strings.TrimSpace(host.Status), "offline") {
m.alertManager.HandleDockerHostOffline(host)
return
}
m.alertManager.CheckDockerHost(host)
}
@@ -0,0 +1,52 @@
package monitoring
import (
"testing"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
func TestCheckMockDockerHostAlertsUsesHostLifecycleForOfflineFixtures(t *testing.T) {
manager := alerts.NewManagerWithDataDir(t.TempDir())
t.Cleanup(manager.Stop)
config := manager.GetConfig()
config.Enabled = true
config.ActivationState = alerts.ActivationPending
config.TimeThresholds = map[string]int{}
config.DockerDefaults.StateDisableConnectivity = false
manager.UpdateConfig(config)
host := models.DockerHost{
ID: "branch-edge",
Hostname: "branch-edge-01",
DisplayName: "Branch Edge",
Status: "offline",
Containers: []models.DockerContainer{
{ID: "portal-1", Name: "branch-portal", State: "exited", Status: "Exited (137)"},
{ID: "sync-1", Name: "branch-syncthing", State: "exited", Status: "Exited (137)"},
{ID: "vpn-1", Name: "branch-vpn", State: "exited", Status: "Exited (137)"},
},
}
// Prove the fixture would create the child storm if it were incorrectly
// treated as fresh online telemetry.
manager.CheckDockerHost(host)
manager.CheckDockerHost(host)
if got := manager.GetActiveAlerts(); len(got) != len(host.Containers) {
t.Fatalf("fresh container evaluation created %d alerts, want %d", len(got), len(host.Containers))
}
monitor := &Monitor{alertManager: manager}
monitor.checkMockDockerHostAlerts(host)
monitor.checkMockDockerHostAlerts(host)
monitor.checkMockDockerHostAlerts(host)
active := manager.GetActiveAlerts()
if len(active) != 1 {
t.Fatalf("offline mock host produced %d active alerts, want one host incident: %+v", len(active), active)
}
if active[0].Type != "docker-host-offline" || active[0].ResourceID != "docker:branch-edge" {
t.Fatalf("offline mock host alert = %+v, want canonical host connectivity incident", active[0])
}
}